Merge branch 'master' into experimental

Correct typo in check for null device

See merge request nvidia/container-toolkit/container-toolkit!101

.common-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,18 +12,16 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 default:
-  image: docker
+  image: docker:stable
   services:
-    - name: docker:dind
+    - name: docker:stable-dind
       command: ["--experimental"]
 
 variables:
   GIT_SUBMODULE_STRATEGY: recursive
   BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}"
-  BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
-  - trigger
   - image
   - lint
   - go-checks
@@ -34,100 +32,45 @@ stages:
   - test
   - scan
   - release
-
-.pipeline-trigger-rules:
-  rules:
-    # We trigger the pipeline if started manually
-    - if: $CI_PIPELINE_SOURCE == "web"
-    # We trigger the pipeline on the main branch
-    - if: $CI_COMMIT_BRANCH == "main"
-    # We trigger the pipeline on the release- branches
-    - if: $CI_COMMIT_BRANCH =~ /^release-.*$/
-    # We trigger the pipeline on tags
-    - if: $CI_COMMIT_TAG && $CI_COMMIT_TAG != ""
-
-workflow:
-  rules:
-    # We trigger the pipeline on a merge request
-    - if: $CI_PIPELINE_SOURCE == 'merge_request_event'
-    # We then add all the regular triggers
-    - !reference [.pipeline-trigger-rules, rules]
-
-# The main or manual job is used to filter out distributions or architectures that are not required on
-# every build.
-.main-or-manual:
-  rules:
-    - !reference [.pipeline-trigger-rules, rules]
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: manual
-
-# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
-trigger-pipeline:
-  stage: trigger
-  script:
-    - echo "starting pipeline"
-  rules:
-    - !reference [.main-or-manual, rules]
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      allow_failure: false
-    - when: always
+  - build-all
 
 # Define the distribution targets
 .dist-amazonlinux2:
-  rules:
-    - !reference [.main-or-manual, rules]
   variables:
     DIST: amazonlinux2
-    PACKAGE_REPO_TYPE: rpm
 
 .dist-centos7:
-  rules:
-    - !reference [.main-or-manual, rules]
   variables:
     DIST: centos7
-    CVE_UPDATES: "cyrus-sasl-lib"
-    PACKAGE_REPO_TYPE: rpm
+    CVE_UPDATES: "nss"
 
 .dist-centos8:
   variables:
     DIST: centos8
-    CVE_UPDATES: "cyrus-sasl-lib"
-    PACKAGE_REPO_TYPE: rpm
 
 .dist-debian10:
-  rules:
-    - !reference [.main-or-manual, rules]
   variables:
     DIST: debian10
-    PACKAGE_REPO_TYPE: debian
+
+.dist-debian9:
+  variables:
+    DIST: debian9
 
 .dist-opensuse-leap15.1:
-  rules:
-    - !reference [.main-or-manual, rules]
   variables:
     DIST: opensuse-leap15.1
-    PACKAGE_REPO_TYPE: rpm
 
 .dist-ubi8:
-  rules:
-    - !reference [.main-or-manual, rules]
   variables:
     DIST: ubi8
-    CVE_UPDATES: "cyrus-sasl-lib"
-    PACKAGE_REPO_TYPE: rpm
+
+.dist-ubuntu16.04:
+  variables:
+    DIST: ubuntu16.04
 
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
-    PACKAGE_REPO_TYPE: debian
-
-.dist-ubuntu20.04:
-  variables:
-    DIST: ubuntu20.04
-    CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
-    PACKAGE_REPO_TYPE: debian
 
 .dist-packaging:
   variables:
@@ -147,8 +90,6 @@ trigger-pipeline:
     ARCH: arm64
 
 .arch-ppc64le:
-  rules:
-    - !reference [.main-or-manual, rules]
   variables:
     ARCH: ppc64le
 
@@ -156,15 +97,6 @@ trigger-pipeline:
   variables:
     ARCH: x86_64
 
-# Define the platform targets
-.platform-amd64:
-  variables:
-    PLATFORM: linux/amd64
-
-.platform-arm64:
-  variables:
-    PLATFORM: linux/arm64
-
 # Define test helpers
 .integration:
   stage: test
@@ -186,30 +118,20 @@ test-packaging:
   needs:
     - image-packaging
 
-# Download the regctl binary for use in the release steps
-.regctl-setup:
-  before_script:
-    - export REGCTL_VERSION=v0.4.5
-    - apk add --no-cache curl
-    - mkdir -p bin
-    - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
-    - chmod a+x bin/regctl
-    - export PATH=$(pwd)/bin:${PATH}
-
 # .release forms the base of the deployment jobs which push images to the CI registry.
 # This is extended with the version to be deployed (e.g. the SHA or TAG) and the
 # target os.
 .release:
-  stage: release
+  stage:
+    release
   variables:
     # Define the source image for the release
     IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     VERSION: "${CI_COMMIT_SHORT_SHA}"
     # OUT_IMAGE_VERSION is overridden for external releases
     OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
+  stage: release
   before_script:
-    - !reference [.regctl-setup, before_script]
-
     # We ensure that the OUT_IMAGE_VERSION is set
     - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
 
@@ -217,16 +139,16 @@ test-packaging:
     # need to tag the image.
     # Note: a leading 'v' is stripped from the version if present
     - apk add --no-cache make bash
-  script:
-    # Log in to the "output" registry, tag the image and push the image
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
-    - regctl registry login "${CI_REGISTRY}" -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}"
-    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || echo "Logging in to output registry ${OUT_REGISTRY}"'
-    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"'
-
-    # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the
-    # Target
-    - make -f build/container/Makefile push-${DIST}
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
+  script:
+    - docker tag "${IMAGE_NAME}:${VERSION}-${DIST}" "${OUT_IMAGE_NAME}:${OUT_IMAGE_VERSION}-${DIST}"
+    # Log in to the "output" registry, tag the image and push the image
+    - 'echo "Logging in to output registry ${OUT_REGISTRY}"'
+    - docker logout
+    - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}"
+    - make IMAGE_NAME=${OUT_IMAGE_NAME} VERSION=${OUT_IMAGE_VERSION} -f build/container/Makefile push-${DIST}
 
 # Define a staging release step that pushes an image to an internal "staging" repository
 # This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
@@ -241,7 +163,7 @@ test-packaging:
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit"
 
 # Define an external release step that pushes an image to an external repository.
-# This includes a devlopment image off main.
+# This includes a devlopment image off master.
 .release:external:
   extends:
     - .release
@@ -261,6 +183,13 @@ release:staging-centos7:
   needs:
     - image-centos7
 
+release:staging-centos8:
+  extends:
+    - .release:staging
+    - .dist-centos8
+  needs:
+    - image-centos8
+
 release:staging-ubi8:
   extends:
     - .release:staging
@@ -268,15 +197,15 @@ release:staging-ubi8:
   needs:
     - image-ubi8
 
-release:staging-ubuntu20.04:
+release:staging-ubuntu18.04:
   extends:
     - .release:staging
-    - .dist-ubuntu20.04
+    - .dist-ubuntu18.04
   needs:
-    - test-toolkit-ubuntu20.04
-    - test-containerd-ubuntu20.04
-    - test-crio-ubuntu20.04
-    - test-docker-ubuntu20.04
+    - test-toolkit-ubuntu18.04
+    - test-containerd-ubuntu18.04
+    - test-crio-ubuntu18.04
+    - test-docker-ubuntu18.04
 
 release:staging-packaging:
   extends:

.github/workflows/blossom-ci.yaml

@@ -1,113 +0,0 @@
-# Copyright (c) 2020-2023, NVIDIA CORPORATION.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# A workflow to trigger ci on hybrid infra (github + self hosted runner)
-name: Blossom-CI
-on:
-  issue_comment:
-    types: [created]
-  workflow_dispatch:
-      inputs:
-          platform:
-            description: 'runs-on argument'     
-            required: false
-          args:
-            description: 'argument'     
-            required: false
-jobs:
-  Authorization:
-    name: Authorization
-    runs-on: blossom 
-    outputs:
-      args: ${{ env.args }}
-     
-    # This job only runs for pull request comments
-    if: |
-         contains( '\
-         anstockatnv,\
-         rohitrajani2018,\
-         cdesiniotis,\
-         shivamerla,\
-         ArangoGutierrez,\
-         elezar,\
-         klueska,\
-         zvonkok,\
-         ', format('{0},', github.actor)) && 
-         github.event.comment.body == '/blossom-ci'  
-    steps:
-      - name: Check if comment is issued by authorized person
-        run: blossom-ci
-        env:
-          OPERATION: 'AUTH'
-          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
-      
-  Vulnerability-scan:
-    name: Vulnerability scan
-    needs: [Authorization]
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v2
-        with:
-          repository: ${{ fromJson(needs.Authorization.outputs.args).repo }}
-          ref: ${{ fromJson(needs.Authorization.outputs.args).ref }}
-          lfs: 'true'
-      
-      # repo specific steps 
-      #- name: Setup java
-      #  uses: actions/setup-java@v1
-      #  with:
-      #    java-version: 1.8
-      
-      # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file
-      #- name: Setup blackduck properties
-      #  run: |
-      #       PROJECTS=$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g')
-      #       echo detect.maven.build.command="-pl=$PROJECTS -am" >> application.properties
-      #       echo detect.maven.included.scopes=compile >> application.properties
-          
-      - name: Run blossom action
-        uses: NVIDIA/blossom-action@main
-        env:
-          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-          REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }}
-        with:
-          args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }}
-          args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }}
-          args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }}
-          
-  Job-trigger:
-    name: Start ci job
-    needs: [Vulnerability-scan]
-    runs-on: blossom
-    steps:
-      - name: Start ci job
-        run: blossom-ci
-        env:
-          OPERATION: 'START-CI-JOB'
-          CI_SERVER: ${{ secrets.CI_SERVER }}
-          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-              
-  Upload-Log:
-    name: Upload log
-    runs-on: blossom
-    if : github.event_name == 'workflow_dispatch'
-    steps:
-      - name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here)
-        run: blossom-ci
-        env:
-          OPERATION: 'POST-PROCESSING'
-          CI_SERVER: ${{ secrets.CI_SERVER }}
-          REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -1,13 +1,8 @@
 dist
-artifacts
 *.swp
 *.swo
-/coverage.out*
+/coverage.out
 /test/output/
 /nvidia-container-runtime
-/nvidia-container-runtime.*
-/nvidia-container-runtime-hook
 /nvidia-container-toolkit
-/nvidia-ctk
 /shared-*
-/release-*

.gitlab-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -94,9 +94,8 @@ unit-tests:
     - .multi-arch-build
     - .package-artifacts
   stage: package-build
-  timeout: 3h
   script:
-    - ./scripts/build-packages.sh ${DIST}-${ARCH}
+    - ./scripts/release.sh ${DIST}-${ARCH}
 
   artifacts:
     name: ${ARTIFACTS_NAME}
@@ -116,6 +115,12 @@ package-amazonlinux2-x86_64:
     - .dist-amazonlinux2
     - .arch-x86_64
 
+package-centos7-ppc64le:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-ppc64le
+
 package-centos7-x86_64:
   extends:
     - .package-build
@@ -146,12 +151,30 @@ package-debian10-amd64:
     - .dist-debian10
     - .arch-amd64
 
+package-debian9-amd64:
+  extends:
+    - .package-build
+    - .dist-debian9
+    - .arch-amd64
+
 package-opensuse-leap15.1-x86_64:
   extends:
     - .package-build
     - .dist-opensuse-leap15.1
     - .arch-x86_64
 
+package-ubuntu16.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-amd64
+
+package-ubuntu16.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-ppc64le
+
 package-ubuntu18.04-amd64:
   extends:
     - .package-build
@@ -170,33 +193,19 @@ package-ubuntu18.04-ppc64le:
     - .dist-ubuntu18.04
     - .arch-ppc64le
 
-.buildx-setup:
-  before_script:
-    -  export BUILDX_VERSION=v0.6.3
-    -  apk add --no-cache curl
-    -  mkdir -p ~/.docker/cli-plugins
-    -  curl -sSLo ~/.docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
-    -  chmod a+x ~/.docker/cli-plugins/docker-buildx
-
-    -  docker buildx create --use --platform=linux/amd64,linux/arm64
-
-    -  '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'
-
 # Define the image build targets
 .image-build:
   stage: image-build
   variables:
     IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     VERSION: "${CI_COMMIT_SHORT_SHA}"
-    PUSH_ON_BUILD: "true"
   before_script:
-    - !reference [.buildx-setup, before_script]
-
-    - apk add --no-cache bash make git
+    - apk add --no-cache bash make
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
   script:
     - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
 
 image-centos7:
   extends:
@@ -204,29 +213,38 @@ image-centos7:
     - .package-artifacts
     - .dist-centos7
   needs:
+    - package-centos7-ppc64le
     - package-centos7-x86_64
 
+image-centos8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos8
+  needs:
+    - package-centos8-aarch64
+    - package-centos8-x86_64
+    - package-centos8-ppc64le
+
 image-ubi8:
   extends:
     - .image-build
     - .package-artifacts
     - .dist-ubi8
   needs:
-    # Note: The ubi8 image uses the centos8 packages
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
+    # Note: The ubi8 image currently uses the centos7 packages
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
 
-image-ubuntu20.04:
+image-ubuntu18.04:
   extends:
     - .image-build
     - .package-artifacts
-    - .dist-ubuntu20.04
+    - .dist-ubuntu18.04
   needs:
     - package-ubuntu18.04-amd64
     - package-ubuntu18.04-arm64
-    - job: package-ubuntu18.04-ppc64le
-      optional: true
+    - package-ubuntu18.04-ppc64le
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
@@ -235,24 +253,21 @@ image-packaging:
     - .package-artifacts
     - .dist-packaging
   needs:
-    - job: package-centos8-aarch64
-    - job: package-centos8-x86_64
-    - job: package-ubuntu18.04-amd64
-    - job: package-ubuntu18.04-arm64
-    - job: package-amazonlinux2-aarch64
-      optional: true
-    - job: package-amazonlinux2-x86_64
-      optional: true
-    - job: package-centos7-x86_64
-      optional: true
-    - job: package-centos8-ppc64le
-      optional: true
-    - job: package-debian10-amd64
-      optional: true
-    - job: package-opensuse-leap15.1-x86_64
-      optional: true
-    - job: package-ubuntu18.04-ppc64le
-      optional: true
+    - package-amazonlinux2-aarch64
+    - package-amazonlinux2-x86_64
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
+    - package-centos8-aarch64
+    - package-centos8-ppc64le
+    - package-centos8-x86_64
+    - package-debian10-amd64
+    - package-debian9-amd64
+    - package-opensuse-leap15.1-x86_64
+    - package-ubuntu16.04-amd64
+    - package-ubuntu16.04-ppc64le
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
 
 # Define publish test helpers
 .test:toolkit:
@@ -284,31 +299,74 @@ image-packaging:
     TEST_CASES: "crio"
 
 # Define the test targets
-test-toolkit-ubuntu20.04:
+test-toolkit-ubuntu18.04:
   extends:
     - .test:toolkit
-    - .dist-ubuntu20.04
+    - .dist-ubuntu18.04
   needs:
-    - image-ubuntu20.04
+    - image-ubuntu18.04
 
-test-containerd-ubuntu20.04:
+test-containerd-ubuntu18.04:
   extends:
     - .test:containerd
-    - .dist-ubuntu20.04
+    - .dist-ubuntu18.04
   needs:
-    - image-ubuntu20.04
+    - image-ubuntu18.04
 
-test-crio-ubuntu20.04:
+test-crio-ubuntu18.04:
   extends:
     - .test:crio
-    - .dist-ubuntu20.04
+    - .dist-ubuntu18.04
   needs:
-    - image-ubuntu20.04
+    - image-ubuntu18.04
 
-test-docker-ubuntu20.04:
+test-docker-ubuntu18.04:
   extends:
     - .test:docker
-    - .dist-ubuntu20.04
+    - .dist-ubuntu18.04
   needs:
-    - image-ubuntu20.04
+    - image-ubuntu18.04
 
+# build-all jobs build packages for every OS / ARCH combination we support.
+#
+# They are run under two conditions:
+# 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0)
+# 2) Manually by a reviewer just before merging a MR.
+.build-all-for-arch:
+  variables:
+    # Setting DIST=docker invokes the docker- release targets
+    DIST: docker
+  extends:
+    - .package-build
+  stage: build-all
+  timeout: 2h 30m
+  rules:
+    - if: $CI_COMMIT_TAG
+      when: always
+
+# The full set of build-all jobs organized to
+# have builds for each ARCH run in parallel.
+build-all-amd64:
+  extends:
+    - .build-all-for-arch
+    - .arch-amd64
+
+build-all-x86_64:
+  extends:
+    - .build-all-for-arch
+    - .arch-x86_64
+
+build-all-ppc64le:
+  extends:
+    - .build-all-for-arch
+    - .arch-ppc64le
+
+build-all-arm64:
+  extends:
+    - .build-all-for-arch
+    - .arch-arm64
+
+build-all-aarch64:
+  extends:
+    - .build-all-for-arch
+    - .arch-aarch64

.gitmodules

@@ -1,12 +1,9 @@
 [submodule "third_party/libnvidia-container"]
 	path = third_party/libnvidia-container
 	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
-	branch = main
 [submodule "third_party/nvidia-container-runtime"]
 	path = third_party/nvidia-container-runtime
 	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
-	branch = main
 [submodule "third_party/nvidia-docker"]
 	path = third_party/nvidia-docker
 	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git
-	branch = main

.nvidia-ci.yml

@@ -1,4 +1,4 @@
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,16 +27,14 @@ default:
 variables:
   DOCKER_DRIVER: overlay2
   DOCKER_TLS_CERTDIR: "/certs"
-  # Release "devel"-tagged images off the main branch
-  RELEASE_DEVEL_BRANCH: "main"
+  # Release "devel"-tagged images off the master branch
+  RELEASE_DEVEL_BRANCH: "master"
   DEVEL_RELEASE_IMAGE_VERSION: "devel"
   # On the multi-arch builder we don't need the qemu setup.
   SKIP_QEMU_SETUP: "1"
   # Define the public staging registry
   STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
-  ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
-  KITMAKER_RELEASE_FOLDER: "kitmaker"
 
 .image-pull:
   stage: image-build
@@ -48,11 +46,9 @@ variables:
     OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
     OUT_REGISTRY: "${CI_REGISTRY}"
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    PUSH_MULTIPLE_TAGS: "false"
   # We delay the job start to allow the public pipeline to generate the required images.
-  rules:
-    - when: delayed
-      start_in: 30 minutes
+  when: delayed
+  start_in: 30 minutes
   timeout: 30 minutes
   retry:
     max: 2
@@ -60,34 +56,39 @@ variables:
       - job_execution_timeout
       - stuck_or_timeout_failure
   before_script:
-    - !reference [.regctl-setup, before_script]
-    - apk add --no-cache make bash
     - >
-      regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
+      docker pull ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
   script:
-    - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
-    - make -f build/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
+    - docker pull ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}
+    - docker tag ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} ${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST}
+    - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}"
+    - docker push ${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST}
 
 image-centos7:
   extends:
-    - .dist-centos7
     - .image-pull
+    - .dist-centos7
+
+image-centos8:
+  extends:
+    - .image-pull
+    - .dist-centos8
 
 image-ubi8:
   extends:
+    - .image-pull
     - .dist-ubi8
-    - .image-pull
 
-image-ubuntu20.04:
+image-ubuntu18.04:
   extends:
-    - .dist-ubuntu20.04
     - .image-pull
+    - .dist-ubuntu18.04
 
 # The DIST=packaging target creates an image containing all built packages
 image-packaging:
   extends:
-    - .dist-packaging
     - .image-pull
+    - .dist-packaging
 
 # We skip the integration tests for the internal CI:
 .integration:
@@ -104,14 +105,14 @@ image-packaging:
   image: "${PULSE_IMAGE}"
   variables:
     IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
-  rules:
-    - if: $SKIP_SCANS != "yes"
-    - when: manual
+    IMAGE_ARCHIVE: "container-toolkit.tar"
+  except:
+    variables:
+      - $SKIP_SCANS && $SKIP_SCANS == "yes"
   before_script:
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
     # TODO: We should specify the architecture here and scan all architectures
-    - docker pull --platform="${PLATFORM}" "${IMAGE}"
+    - docker pull "${IMAGE}"
     - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
     - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
     - >
@@ -130,57 +131,34 @@ image-packaging:
       - policy_evaluation.json
 
 # Define the scan targets
-scan-centos7-amd64:
+scan-centos7:
   extends:
-    - .dist-centos7
-    - .platform-amd64
     - .scan
+    - .dist-centos7
   needs:
     - image-centos7
 
-scan-centos7-arm64:
+scan-centos8:
   extends:
-    - .dist-centos7
-    - .platform-arm64
     - .scan
+    - .dist-centos8
   needs:
-    - image-centos7
-    - scan-centos7-amd64
+    - image-centos8
 
-scan-ubuntu20.04-amd64:
+scan-ubuntu18.04:
   extends:
-    - .dist-ubuntu20.04
-    - .platform-amd64
     - .scan
+    - .dist-ubuntu18.04
   needs:
-    - image-ubuntu20.04
+    - image-ubuntu18.04
 
-scan-ubuntu20.04-arm64:
+scan-ubi8:
   extends:
-    - .dist-ubuntu20.04
-    - .platform-arm64
     - .scan
-  needs:
-    - image-ubuntu20.04
-    - scan-ubuntu20.04-amd64
-
-scan-ubi8-amd64:
-  extends:
     - .dist-ubi8
-    - .platform-amd64
-    - .scan
   needs:
     - image-ubi8
 
-scan-ubi8-arm64:
-  extends:
-    - .dist-ubi8
-    - .platform-arm64
-    - .scan
-  needs:
-    - image-ubi8
-    - scan-ubi8-amd64
-
 # Define external release helpers
 .release:ngc:
   extends:
@@ -191,51 +169,61 @@ scan-ubi8-arm64:
     OUT_REGISTRY: "${NGC_REGISTRY}"
     OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
 
-.release:packages:
-  stage: release
-  needs:
-    - image-packaging
-  variables:
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-    PACKAGE_REGISTRY: "${CI_REGISTRY}"
-    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
-    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
-    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
-    KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
-  script:
-    - !reference [.regctl-setup, before_script]
-    - apk add --no-cache bash git
-    - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
-    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
-    # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
-    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
-
-# Define the package release targets
-release:packages:kitmaker:
+.release:dockerhub:
   extends:
-    - .release:packages
+    - .release:external
+  variables:
+    OUT_REGISTRY_USER: "${REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${REGISTRY_TOKEN}"
+    OUT_REGISTRY: "${DOCKERHUB_REGISTRY}"
+    OUT_IMAGE_NAME: "${REGISTRY_IMAGE}"
 
-release:staging-ubuntu20.04:
+release:staging-ubuntu18.04:
   extends:
     - .release:staging
-    - .dist-ubuntu20.04
+    - .dist-ubuntu18.04
   needs:
-    - image-ubuntu20.04
+    - image-ubuntu18.04
 
 # Define the external release targets
 # Release to NGC
 release:ngc-centos7:
   extends:
+    - .release:ngc
     - .dist-centos7
-    - .release:ngc
 
-release:ngc-ubuntu20.04:
+release:ngc-centos8:
   extends:
-    - .dist-ubuntu20.04
     - .release:ngc
+    - .dist-centos8
+
+release:ngc-ubuntu18:
+  extends:
+    - .release:ngc
+    - .dist-ubuntu18.04
 
 release:ngc-ubi8:
   extends:
-    - .dist-ubi8
     - .release:ngc
+    - .dist-ubi8
+
+# Release to Dockerhub
+release:dockerhub-centos7:
+  extends:
+    - .release:dockerhub
+    - .dist-centos7
+
+release:dockerhub-centos8:
+  extends:
+    - .release:dockerhub
+    - .dist-centos8
+
+release:dockerhub-ubuntu18:
+  extends:
+    - .release:dockerhub
+    - .dist-ubuntu18.04
+
+release:dockerhub-ubi8:
+  extends:
+    - .release:dockerhub
+    - .dist-ubi8

CHANGELOG.md

@@ -1,293 +0,0 @@
-# NVIDIA Container Toolkit Changelog
-
-## v1.13.0-rc.3
-
-* Only initialize NVML for modes that require it when runing `nvidia-ctk cdi generate`.
-* Prefer /run over /var/run when locating nvidia-persistenced and nvidia-fabricmanager sockets.
-* Fix the generation of CDI specifications for management containers when the driver libraries are not in the LDCache.
-* Add transformers to deduplicate and simplify CDI specifications.
-* Generate a simplified CDI specification by default. This means that entities in the common edits in a spec are not included in device definitions.
-* Also return an error from the nvcdi.New constructor instead of panicing.
-* Detect XOrg libraries for injection and CDI spec generation.
-* Add `nvidia-container-runtime.modes.cdi.annotation-prefixes` config option that allows the CDI annotation prefixes that are read to be overridden.
-
-* [libnvidia-container] Fix segmentation fault when RPC initialization fails.
-* [libnvidia-container] Build centos variants of the NVIDIA Container Library with static libtirpc v1.3.2.
-* [libnvidia-container] Remove make targets for fedora35 as the centos8 packages are compatible.
-
-## v1.13.0-rc.2
-
-* Don't fail chmod hook if paths are not injected
-* Only create `by-path` symlinks if CDI devices are actually requested.
-* Fix possible blank `nvidia-ctk` path in generated CDI specifications
-* Fix error in postun scriplet on RPM-based systems
-* Only check `NVIDIA_VISIBLE_DEVICES` for environment variables if no annotations are specified.
-* Add `cdi.default-kind` config option for constructing fully-qualified CDI device names in CDI mode
-* Add support for `accept-nvidia-visible-devices-envvar-unprivileged` config setting in CDI mode
-* Add `nvidia-container-runtime-hook.skip-mode-detection` config option to bypass mode detection. This allows `legacy` and `cdi` mode, for example, to be used at the same time.
-* Add support for generating CDI specifications for GDS and MOFED devices
-* Ensure CDI specification is validated on save when generating a spec
-* Rename `--discovery-mode` argument to `--mode` for `nvidia-ctk cdi generate`
-* [libnvidia-container] Fix segfault on WSL2 systems
-* [toolkit-container] Add `--cdi-enabled` flag to toolkit config
-* [toolkit-container] Install `nvidia-ctk` from toolkit container
-* [toolkit-container] Use installed `nvidia-ctk` path in NVIDIA Container Toolkit config
-* [toolkit-container] Bump CUDA base images to 12.1.0
-* [toolkit-container] Set `nvidia-ctk` path in the
-* [toolkit-container] Add `cdi.k8s.io/*` to set of allowed annotations in containerd config
-* [toolkit-container] Generate CDI specification for use in management containers
-* [toolkit-container] Install experimental runtime as `nvidia-container-runtime.experimental` instead of `nvidia-container-runtime-experimental`
-* [toolkit-container] Install and configure mode-specific runtimes for `cdi` and `legacy` modes
-
-## v1.13.0-rc.1
-
-* Include MIG-enabled devices as GPUs when generating CDI specification
-* Fix missing NVML symbols when running `nvidia-ctk` on some platforms [#49]
-* Add CDI spec generation for WSL2-based systems to `nvidia-ctk cdi generate` command
-* Add `auto` mode to `nvidia-ctk cdi generate` command to automatically detect a WSL2-based system over a standard NVML-based system.
-* Add mode-specific (`.cdi` and `.legacy`) NVIDIA Container Runtime binaries for use in the GPU Operator
-* Discover all `gsb*.bin` GSP firmware files when generating CDI specification.
-* Align `.deb` and `.rpm` release candidate package versions
-* Remove `fedora35` packaging targets
-* [libnvidia-container] Include all `gsp*.bin` firmware files if present
-* [libnvidia-container] Align `.deb` and `.rpm` release candidate package versions
-* [libnvidia-container] Remove `fedora35` packaging targets
-* [toolkit-container] Install `nvidia-container-toolkit-operator-extensions` package for mode-specific executables.
-* [toolkit-container] Allow `nvidia-container-runtime.mode` to be set when configuring the NVIDIA Container Toolkit
-
-## v1.12.0
-
-* Promote `v1.12.0-rc.5` to `v1.12.0`
-* Rename `nvidia cdi generate` `--root` flag to `--driver-root` to better indicate intent
-* [libnvidia-container] Add nvcubins.bin to DriverStore components under WSL2
-* [toolkit-container] Bump CUDA base images to 12.0.1
-
-## v1.12.0-rc.5
-
-* Fix bug here the `nvidia-ctk` path was not properly resolved. This causes failures to run containers when the runtime is configured in `csv` mode or if the `NVIDIA_DRIVER_CAPABILITIES` includes `graphics` or `display` (e.g. `all`).
-
-## v1.12.0-rc.4
-
-* Generate a minimum CDI spec version for improved compatibility.
-* Add `--device-name-strategy` options to the `nvidia-ctk cdi generate` command that can be used to control how device names are constructed.
-* Set default for CDI device name generation to `index` to generate device names such as `nvidia.com/gpu=0` or `nvidia.com/gpu=1:0` by default.
-
-## v1.12.0-rc.3
-
-* Don't fail if by-path symlinks for DRM devices do not exist
-* Replace the --json flag with a --format [json|yaml] flag for the nvidia-ctk cdi generate command
-* Ensure that the CDI output folder is created if required
-* When generating a CDI specification use a blank host path for devices to ensure compatibility with the v0.4.0 CDI specification
-* Add injection of Wayland JSON files
-* Add GSP firmware paths to generated CDI specification
-* Add --root flag to nvidia-ctk cdi generate command
-
-## v1.12.0-rc.2
-
-* Inject Direct Rendering Manager (DRM) devices into a container using the NVIDIA Container Runtime
-* Improve logging of errors from the NVIDIA Container Runtime
-* Improve CDI specification generation to support rootless podman
-* Use `nvidia-ctk cdi generate` to generate CDI specifications instead of `nvidia-ctk info generate-cdi`
-* [libnvidia-container] Skip creation of existing files when these are already mounted
-
-## v1.12.0-rc.1
-
-* Add support for multiple Docker Swarm resources
-* Improve injection of Vulkan configurations and libraries
-* Add `nvidia-ctk info generate-cdi` command to generated CDI specification for available devices
-* [libnvidia-container] Include NVVM compiler library in compute libs
-
-## v1.11.0
-
-* Promote v1.11.0-rc.3 to v1.11.0
-
-## v1.11.0-rc.3
-
-* Build fedora35 packages
-* Introduce an `nvidia-container-toolkit-base` package for better dependency management
-* Fix removal of `nvidia-container-runtime-hook` on RPM-based systems
-* Inject platform files into container on Tegra-based systems
-* [toolkit container] Update CUDA base images to 11.7.1
-* [libnvidia-container] Preload libgcc_s.so.1 on arm64 systems
-
-## v1.11.0-rc.2
-
-* Allow `accept-nvidia-visible-devices-*` config options to be set by toolkit container
-* [libnvidia-container] Fix bug where LDCache was not updated when the `--no-pivot-root` option was specified
-
-## v1.11.0-rc.1
-
-* Add discovery of GPUDirect Storage (`nvidia-fs*`) devices if the `NVIDIA_GDS` environment variable of the container is set to `enabled`
-* Add discovery of MOFED Infiniband devices if the `NVIDIA_MOFED` environment variable of the container is set to `enabled`
-* Fix bug in CSV mode where libraries listed as `sym` entries in mount specification are not added to the LDCache.
-* Rename `nvidia-container-toolkit` executable to `nvidia-container-runtime-hook` and create `nvidia-container-toolkit` as a symlink to `nvidia-container-runtime-hook` instead.
-* Add `nvidia-ctk runtime configure` command to configure the Docker config file (e.g. `/etc/docker/daemon.json`) for use with the NVIDIA Container Runtime.
-
-## v1.10.0
-
-* Promote v1.10.0-rc.3 to v1.10.0
-
-## v1.10.0-rc.3
-
-* Use default config instead of raising an error if config file cannot be found
-* Ignore NVIDIA_REQUIRE_JETPACK* environment variables for requirement checks
-* Fix bug in detection of Tegra systems where `/sys/devices/soc0/family` is ignored
-* Fix bug where links to devices were detected as devices
-* [libnvida-container] Fix bug introduced when adding libcudadebugger.so to list of libraries
-
-## v1.10.0-rc.2
-
-* Add support for NVIDIA_REQUIRE_* checks for cuda version and arch to csv mode
-* Switch to debug logging to reduce log verbosity
-* Support logging to logs requested in command line
-* Fix bug when launching containers with relative root path (e.g. using containerd)
-* Allow low-level runtime path to be set explicitly as nvidia-container-runtime.runtimes option
-* Fix failure to locate low-level runtime if PATH envvar is unset
-* Replace experimental option for NVIDIA Container Runtime with nvidia-container-runtime.mode = csv option
-* Use csv as default mode on Tegra systems without NVML
-* Add --version flag to all CLIs
-* [libnvidia-container] Bump libtirpc to 1.3.2
-* [libnvidia-container] Fix bug when running host ldconfig using glibc compiled with a non-standard prefix
-* [libnvidia-container] Add libcudadebugger.so to list of compute libraries
-
-## v1.10.0-rc.1
-
-* Include nvidia-ctk CLI in installed binaries
-* Add experimental option to NVIDIA Container Runtime
-
-## v1.9.0
-
-* [libnvidia-container] Add additional check for Tegra in /sys/.../family file in CLI
-* [libnvidia-container] Update jetpack-specific CLI option to only load Base CSV files by default
-* [libnvidia-container] Fix bug (from 1.8.0) when mounting GSP firmware into containers without /lib to /usr/lib symlinks
-* [libnvidia-container] Update nvml.h to CUDA 11.6.1 nvML_DEV 11.6.55
-* [libnvidia-container] Update switch statement to include new brands from latest nvml.h
-* [libnvidia-container] Process all --require flags on Jetson platforms
-* [libnvidia-container] Fix long-standing issue with running ldconfig on Debian systems
-
-## v1.8.1
-
-* [libnvidia-container] Fix bug in determining cgroup root when running in nested containers
-* [libnvidia-container] Fix permission issue when determining cgroup version
-
-## v1.8.0
-
-* Promote 1.8.0-rc.2-1 to 1.8.0
-
-## v1.8.0-rc.2
-
-* Remove support for building amazonlinux1 packages
-
-## v1.8.0-rc.1
-
-* [libnvidia-container] Add support for cgroupv2
-* Release toolkit-container images from nvidia-container-toolkit repository
-
-## v1.7.0
-
-* Promote 1.7.0-rc.1-1 to 1.7.0
- * Bump Golang version to 1.16.4
-
-## v1.7.0-rc.1
-
-* Specify containerd runtime type as string in config tools to remove dependency on containerd package
-* Add supported-driver-capabilities config option to allow for a subset of all driver capabilities to be specified
-
-## v1.6.0
-
-* Promote 1.6.0-rc.3-1 to 1.6.0
- * Fix unnecessary logging to stderr instead of configured nvidia-container-runtime log file
-
-## v1.6.0-rc.3
-
-* Add supported-driver-capabilities config option to the nvidia-container-toolkit
-* Move OCI and command line checks for runtime to internal oci package
-
-## v1.6.0-rc.2
-
-* Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
-
-## v1.6.0-rc.1
-
-* Add AARCH64 package for Amazon Linux 2
-* Include nvidia-container-runtime into nvidia-container-toolkit package
-
-## v1.5.1
-
-* Fix bug where Docker Swarm device selection is ignored if NVIDIA_VISIBLE_DEVICES is also set
-* Improve unit testing by using require package and adding coverage reports
-* Remove unneeded go dependencies by running go mod tidy
-* Move contents of pkg directory to cmd for CLI tools
-* Ensure make binary target explicitly sets GOOS
-
-## v1.5.0
-
-* Add dependence on libnvidia-container-tools >= 1.4.0
-* Add golang check targets to Makefile
-* Add Jenkinsfile definition for build targets
-* Move docker.mk to docker folder
-
-## v1.4.2
-
-* Add dependence on libnvidia-container-tools >= 1.3.3
-
-## v1.4.1
-
-* Ignore NVIDIA_VISIBLE_DEVICES for containers with insufficent privileges
-* Add dependence on libnvidia-container-tools >= 1.3.2
-
-## v1.4.0
-
-* Add 'compute' capability to list of defaults
-* Add dependence on libnvidia-container-tools >= 1.3.1
-
-## v1.3.0
-
-* Promote 1.3.0-rc.2-1 to 1.3.0
-* Add dependence on libnvidia-container-tools >= 1.3.0
-
-## v1.3.0-rc.2
-
-* 2c180947 Add more tests for new semantics with device list from volume mounts
-* 7c003857 Refactor accepting device lists from volume mounts as a boolean
-
-## v1.3.0-rc.1
-
-* b50d86c1 Update build system to accept a TAG variable for things like rc.x
-* fe65573b Add common CI tests for things like golint, gofmt, unit tests, etc.
-* da6fbb34 Revert "Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*"
-* a7fb3330 Flip build-all targets to run automatically on merge requests
-* 8b248b66 Rename github.com/NVIDIA/container-toolkit to nvidia-container-toolkit
-* da36874e Add new config options to pull device list from mounted files instead of ENVVAR
-
-## v1.2.1
-
-* 4e6e0ed4 Add 'ngx' to list of*all* driver capabilities
-* 2f4af743 List config.toml as a config file in the RPM SPEC
-
-## v1.2.0
-
-*  8e0aab46 Fix repo listed in changelog for debian distributions
-*  320bb6e4 Update dependence on libnvidia-container to 1.2.0
-*  6cfc8097 Update package license to match source license
-*  e7dc3cbb Fix debian copyright file
-*  d3aee3e0 Add the 'ngx' driver capability
-
-## v1.1.2
-
-* c32237f3 Add support for parsing Linux Capabilities for older OCI specs
-
-## v1.1.1
-
-* d202aded Update dependence to libnvidia-container 1.1.1
-
-## v1.1.0
-
-* 4e4de762 Update build system to support multi-arch builds
-* fcc1d116 Add support for MIG (Multi-Instance GPUs)
-* d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*
-* 60f165ad Add no-pivot option to toolkit
-
-## v1.0.5
-
-* Initial release. Replaces older package nvidia-container-runtime-hook. (Closes: #XXXXXX)

DEVELOPMENT.md

@@ -13,7 +13,7 @@ The `nvidia-container-toolkit` resides in this repo directly.
 
 In oder to build the packages, the following command is executed
 ```sh
-./scripts/build-packages.sh TARGET
+./scripts/build-all-components.sh TARGET
 ```
 where `TARGET` is a make target that is valid for each of the sub-components.
 
@@ -21,8 +21,6 @@ These include:
 * `ubuntu18.04-amd64`
 * `centos8-x86_64`
 
-If no `TARGET` is specified, all valid release targets are built.
-
 The packages are generated in the `dist` folder.
 
 ## Testing local changes
@@ -39,23 +37,9 @@ The [test/release](./test/release/) folder contains documentation on how the ins
 
 ## Releasing
 
-In order to release packages required for a release, a utility script
-[`scripts/release-packages.sh`](./scripts/release-packages.sh) is provided.
-This script can be executed as follows:
-
-```bash
-GPG_LOCAL_USER="GPG_USER" \
-MASTER_KEY_PATH=/path/to/gpg-master.key \
-SUB_KEY_PATH=/path/to/gpg-subkey.key \
-    ./scripts/release-packages.sh REPO PACKAGE_REPO_ROOT [REFERENCE]
+A utility script [`scripts/release.sh`](./scripts/release.sh) is provided to build
+packages required for release. If run without arguments, all supported distribution-architecture combinations are built. A specific distribution-architecture pair can also be provided
+```sh
+./scripts/release.sh ubuntu18.04-amd64
 ```
-
-Where `REPO` is one of `stable` or `experimental`, `PACKAGE_REPO_ROOT` is the local path to the `libnvidia-container` repository checked out to the `gh-pages` branch, and `REFERENCE` is the git SHA that is to be released. If reference is not specified `HEAD` is assumed.
-
-This scripts performs the following basic functions:
-* Pulls the package image defined by the `REFERENCE` git SHA from the staging registry,
-* Copies the required packages to the package repository at `PACKAGE_REPO_ROOT/REPO`,
-* Signs the packages using the specified GPG keys
-
-While the last two are performed, commits are added to the package repository. These can be pushed to the relevant repository.
-
+where the `amd64` builds for `ubuntu18.04` are provided as an example.

Makefile

@@ -38,21 +38,16 @@ EXAMPLE_TARGETS := $(patsubst %,example-%, $(EXAMPLES))
 CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
 CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
 
+$(info CMD_TARGETS=$(CMD_TARGETS))
+
 CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
-MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate licenses $(CHECK_TARGETS)
+MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
 
 TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
 
 DOCKER_TARGETS := $(patsubst %,docker-%, $(TARGETS))
 .PHONY: $(TARGETS) $(DOCKER_TARGETS)
 
-ifeq ($(VERSION),)
-CLI_VERSION = $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
-else
-CLI_VERSION = $(VERSION)
-endif
-CLI_VERSION_PACKAGE = github.com/NVIDIA/nvidia-container-toolkit/internal/info
-
 GOOS ?= linux
 
 binaries: cmds
@@ -61,7 +56,7 @@ cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
 endif
 cmds: $(CMD_TARGETS)
 $(CMD_TARGETS): cmd-%:
-	GOOS=$(GOOS) go build -ldflags "-extldflags=-Wl,-z,lazy -s -w -X $(CLI_VERSION_PACKAGE).gitCommit=$(GIT_COMMIT) -X $(CLI_VERSION_PACKAGE).version=$(CLI_VERSION)" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+	GOOS=$(GOOS) go build -ldflags "-s -w" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
 
 build:
 	GOOS=$(GOOS) go build ./...
@@ -95,7 +90,11 @@ ineffassign:
 
 lint:
 # We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
-	go list -f '{{.Dir}}' $(MODULE)/... | xargs golint -set_exit_status
+	go list -f '{{.Dir}}' $(MODULE)/... | grep -v /internal/ | xargs golint -set_exit_status
+
+lint-internal:
+# We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
+	go list -f '{{.Dir}}' $(MODULE)/internal/... | xargs golint -set_exit_status
 
 misspell:
 	misspell $(MODULE)/...
@@ -103,9 +102,6 @@ misspell:
 vet:
 	go vet $(MODULE)/...
 
-licenses:
-	go-licenses csv $(MODULE)/...
-
 COVERAGE_FILE := coverage.out
 test: build cmds
 	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...
@@ -146,15 +142,3 @@ $(DOCKER_TARGETS): docker-%: .build-image
 		--user $$(id -u):$$(id -g) \
 		$(BUILDIMAGE) \
 			make $(*)
-
-# Start an interactive shell using the development image.
-PHONY: .shell
-.shell:
-	$(DOCKER) run \
-		--rm \
-		-ti \
-		-e GOCACHE=/tmp/.cache \
-		-v $(PWD):$(PWD) \
-		-w $(PWD) \
-		--user $$(id -u):$$(id -g) \
-		$(BUILDIMAGE)

README.md

@@ -1,6 +1,6 @@
 # NVIDIA Container Toolkit
 
-[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-container-toolkit?style=flat-square)](https://raw.githubusercontent.com/NVIDIA/nvidia-container-toolkit/main/LICENSE)
+[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-container-toolkit?style=flat-square)](https://raw.githubusercontent.com/NVIDIA/nvidia-container-toolkit/master/LICENSE)
 [![Documentation](https://img.shields.io/badge/documentation-wiki-blue.svg?style=flat-square)](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/overview.html)
 [![Package repository](https://img.shields.io/badge/packages-repository-b956e8.svg?style=flat-square)](https://nvidia.github.io/libnvidia-container)
 

build/container/Dockerfile.centos

@@ -42,16 +42,6 @@ RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" .
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
-ARG BASE_DIST
-# See https://www.centos.org/centos-linux-eol/
-# and https://stackoverflow.com/a/70930049 for move to vault.centos.org
-# and https://serverfault.com/questions/1093922/failing-to-run-yum-update-in-centos-8 for move to vault.epel.cloud
-RUN [[ "${BASE_DIST}" != "centos8" ]] || \
-    ( \
-      sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \
-      sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* \
-    )
-
 ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
@@ -63,13 +53,11 @@ COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
 WORKDIR /artifacts/packages
 
 ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH ${TARGETARCH}
-RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
-    yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-1.*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-1.*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*-${PACKAGE_VERSION}*.rpm
+ARG PACKAGE_ARCH
+RUN yum localinstall -y \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
 
 WORKDIR /work
 
@@ -85,7 +73,7 @@ LABEL release="N/A"
 LABEL summary="Automatically Configure your Container Runtime for GPU support."
 LABEL description="See summary"
 
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+COPY ./LICENSE /licenses/LICENSE
 
 # Install / upgrade packages here that are required to resolve CVEs
 ARG CVE_UPDATES

build/container/Dockerfile.packaging

@@ -15,27 +15,15 @@
 ARG BASE_DIST
 ARG CUDA_VERSION
 ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
+ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
+
 ARG ARTIFACTS_ROOT
 COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
 
-# build-args are added to the manifest.txt file below.
-ARG BASE_DIST
-ARG PACKAGE_DIST
-ARG PACKAGE_VERSION
-ARG GIT_BRANCH
-ARG GIT_COMMIT
-ARG GIT_COMMIT_SHORT
-ARG SOURCE_DATE_EPOCH
-ARG VERSION
-
-# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
-RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
-    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
-    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+COPY ./LICENSE /licenses/LICENSE

build/container/Dockerfile.ubuntu

@@ -40,15 +40,11 @@ COPY . .
 RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
 
 
-FROM nvcr.io/nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
-
-# Remove the CUDA repository configurations to avoid issues with rotated GPG keys
-RUN rm -f /etc/apt/sources.list.d/cuda.list
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
 ARG DEBIAN_FRONTEND=noninteractive
 RUN apt-get update && apt-get install -y --no-install-recommends \
     libcap2 \
-    curl \
         && \
     rm -rf /var/lib/apt/lists/*
 
@@ -63,21 +59,11 @@ COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
 WORKDIR /artifacts/packages
 
 ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH ${TARGETARCH}
-
-ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container"
-ARG LIBNVIDIA_CONTAINER0_VERSION
-RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \
-        curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \
-            --output ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb && \
-        dpkg -i ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb; \
-    fi
-
+ARG PACKAGE_ARCH
 RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_1.*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_1.*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*_${PACKAGE_VERSION}*.deb
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
 
 WORKDIR /work
 
@@ -93,13 +79,6 @@ LABEL release="N/A"
 LABEL summary="Automatically Configure your Container Runtime for GPU support."
 LABEL description="See summary"
 
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
-
-# Install / upgrade packages here that are required to resolve CVEs
-ARG CVE_UPDATES
-RUN if [ -n "${CVE_UPDATES}" ]; then \
-        apt-get update && apt-get upgrade -y ${CVE_UPDATES} && \
-        rm -rf /var/lib/apt/lists/*; \
-    fi
+COPY ./LICENSE /licenses/LICENSE
 
 ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -12,14 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-BUILD_MULTI_ARCH_IMAGES ?= false
-DOCKER ?= docker
-
-BUILDX =
-ifeq ($(BUILD_MULTI_ARCH_IMAGES),true)
-BUILDX = buildx
-endif
-
+DOCKER   ?= docker
 MKDIR    ?= mkdir
 DIST_DIR ?= $(CURDIR)/dist
 
@@ -32,47 +25,36 @@ IMAGE_NAME := $(REGISTRY)/container-toolkit
 endif
 
 VERSION ?= $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
-IMAGE_VERSION := $(VERSION)
 
 IMAGE_TAG ?= $(VERSION)-$(DIST)
 IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
 
-OUT_IMAGE_NAME ?= $(IMAGE_NAME)
-OUT_IMAGE_VERSION ?= $(IMAGE_VERSION)
-OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
-OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
-
 ##### Public rules #####
-DEFAULT_PUSH_TARGET := ubuntu20.04
-DISTRIBUTIONS := ubuntu20.04 ubi8 centos7
+DEFAULT_PUSH_TARGET := ubuntu18.04
+TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
 
 META_TARGETS := packaging
 
-BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS))
-PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS))
-TEST_TARGETS := $(patsubst %,test-%,$(DISTRIBUTIONS))
+BUILD_TARGETS := $(patsubst %,build-%,$(TARGETS) $(META_TARGETS))
+PUSH_TARGETS := $(patsubst %,push-%,$(TARGETS) $(META_TARGETS))
+TEST_TARGETS := $(patsubst %,test-%, $(TARGETS))
 
-.PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
+.PHONY: $(TARGETS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
-ifneq ($(BUILD_MULTI_ARCH_IMAGES),true)
-include $(CURDIR)/build/container/native-only.mk
-else
-include $(CURDIR)/build/container/multi-arch.mk
-endif
+push-%: DIST = $(*)
+$(PUSH_TARGETS): push-%:
+	$(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)"
 
 # For the default push target we also push a short tag equal to the version.
 # We skip this for the development release
 DEVEL_RELEASE_IMAGE_VERSION ?= devel
-PUSH_MULTIPLE_TAGS ?= true
-ifeq ($(strip $(OUT_IMAGE_VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
-PUSH_MULTIPLE_TAGS = false
-endif
-ifeq ($(PUSH_MULTIPLE_TAGS),true)
+ifneq ($(strip $(VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
 push-$(DEFAULT_PUSH_TARGET): push-short
 endif
+push-short:
+	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(IMAGE_NAME):$(VERSION)"
+	$(DOCKER) push "$(IMAGE_NAME):$(VERSION)"
 
-push-%: DIST = $(*)
-push-short: DIST = $(DEFAULT_PUSH_TARGET)
 
 build-%: DIST = $(*)
 build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
@@ -82,22 +64,17 @@ ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
 # Use a generic build target to build the relevant images
 $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 	DOCKER_BUILDKIT=1 \
-		$(DOCKER) $(BUILDX) build --pull \
-		$(DOCKER_BUILD_OPTIONS) \
-		$(DOCKER_BUILD_PLATFORM_OPTIONS) \
+		$(DOCKER) build --pull \
+		--platform=linux/amd64 \
 		--tag $(IMAGE) \
 		--build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \
 		--build-arg BASE_DIST="$(BASE_DIST)" \
 		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
 		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-		--build-arg LIBNVIDIA_CONTAINER0_VERSION="$(LIBNVIDIA_CONTAINER0_DEPENDENCY)" \
 		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
+		--build-arg PACKAGE_ARCH="$(PACKAGE_ARCH)" \
 		--build-arg VERSION="$(VERSION)" \
-		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
-		--build-arg GIT_COMMIT_SHORT="$(GIT_COMMIT_SHORT)" \
-		--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
-		--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
 		--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
 		-f $(DOCKERFILE) \
 		$(CURDIR)
@@ -105,21 +82,28 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 
 build-ubuntu%: BASE_DIST = $(*)
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
-build-ubuntu%: PACKAGE_DIST = ubuntu18.04
-build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION)
+build-ubuntu%: PACKAGE_ARCH := amd64
+build-ubuntu%: PACKAGE_DIST = $(BASE_DIST)
+build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 
+# TODO: Update this to use the centos8 packages
 build-ubi8: BASE_DIST := ubi8
 build-ubi8: DOCKERFILE_SUFFIX := centos
-build-ubi8: PACKAGE_DIST = centos8
+build-ubi8: PACKAGE_ARCH := x86_64
+build-ubi8: PACKAGE_DIST = centos7
+build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
-build-centos7: BASE_DIST = $(*)
-build-centos7: DOCKERFILE_SUFFIX := centos
-build-centos7: PACKAGE_DIST = $(BASE_DIST)
+build-centos%: BASE_DIST = $(*)
+build-centos%: DOCKERFILE_SUFFIX := centos
+build-centos%: PACKAGE_ARCH := x86_64
+build-centos%: PACKAGE_DIST = $(BASE_DIST)
+build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging
 build-packaging: PACKAGE_ARCH := amd64
 build-packaging: PACKAGE_DIST = all
+build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 
 # Test targets
 test-%: DIST = $(*)
@@ -143,7 +127,10 @@ test-packaging:
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
 	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

build/container/multi-arch.mk

@@ -1,37 +0,0 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-PUSH_ON_BUILD ?= false
-DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD)
-DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64,linux/arm64
-
-REGCTL ?= regctl
-$(PUSH_TARGETS): push-%:
-	$(REGCTL) \
-	        image copy \
-	        $(IMAGE) $(OUT_IMAGE)
-
-push-short:
-	$(REGCTL) \
-	        image copy \
-	        $(IMAGE) $(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)
-
-# We only have x86_64 packages for centos7
-build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
-
-# We only generate amd64 image for ubuntu18.04
-build-ubuntu18.04: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
-
-# We only generate a single image for packaging targets
-build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64

build/container/native-only.mk

@@ -1,23 +0,0 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64
-
-$(PUSH_TARGETS): push-%:
-	$(DOCKER) tag "$(IMAGE)" "$(OUT_IMAGE)"
-	$(DOCKER) push "$(OUT_IMAGE)"
-
-push-short:
-	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"
-	$(DOCKER) push "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"

cmd/nvidia-container-runtime.experimental/README.md

@@ -0,0 +1,170 @@
+# The Experimental NVIDIA Container Runtime
+
+## Introduction
+
+The experimental NVIDIA Container Runtime is a proof-of-concept runtime that
+approaches the problem of making GPUs (or other NVIDIA devices) available in
+containerized environments in a different manner to the existing
+[NVIDIA Container Runtime](../nvidia-container-runtime). Wherease the current
+runtime relies on the [NVIDIA Container Library](https://github.com/NVIDIA/libnvidia-container)
+to perform the modifications to a container, the experiemental runtime aims to
+express the required modifications in terms of changes to a container's [OCI
+runtime specification](https://github.com/opencontainers/runtime-spec). This
+also aligns with open initiatives such as the [Container Device Interface (CDI)](https://github.com/container-orchestrated-devices/container-device-interface).
+
+## Known Limitations
+
+* The path of NVIDIA CUDA libraries / binaries injected into the container currently match that of the host system. This means that on an Ubuntu-based host systems these would be at `/usr/lib/x86_64-linux-gnu`
+even if the container distribution would normally expect these at another location (e.g. `/usr/lib64`)
+* Tools such as `nvidia-smi` may create additional device nodes in the container when run. This is
+prevented in the "classic" runtime (and the NVIDIA Container Library) by modifying
+the `/proc/driver/nvidia/params` file in the container.
+* Other `NVIDIA_*` environment variables (e.g. `NVIDIA_DRIVER_CAPABILITIES`) are
+not considered to filter mounted libraries or binaries. This is equivalent to
+always using `NVIDIA_DRIVER_CAPABILITIES=all`.
+
+## Building / Installing
+
+The experimental NVIDIA Container Runtime is a self-contained golang binary and
+can thus be built from source, or installed directly using `go install`.
+
+### From source
+
+After cloning the `nvidia-container-toolkit` repository from [GitLab](https://gitlab.com/nvidia/container-toolkit/container-toolkit)
+or from the read-only mirror on [GitHub](https://github.com/NVIDIA/nvidia-container-toolkit)
+running the following make command in the repository root:
+```bash
+make cmd-nvidia-container-runtime.experimental
+```
+will create an executable file `nvidia-container-runtime.experimental` in the
+root.
+
+A dockerized target:
+```bash
+make docker-cmd-nvidia-container-runtime.experimental
+```
+will also create the executable file `nvidia-container-runtime.experimental`
+without requiring the setup of a development environment (with the exception)
+of having `make` and `docker` installed.
+
+### Go install
+
+The experimental NVIDIA Container Runtime can also be `go installed` by running
+the following command:
+
+```bash
+go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@experimental
+```
+which will build and install the `nvidia-container-runtime.experimental`
+executable in the `${GOPATH}/bin` folder.
+
+## Using the Runtime
+
+The experimental NVIDIA Container Runtime is intended as a drop-in replacement
+for the "classic" NVIDIA Container Runtime. As such it is used in the same
+way (with the exception of the known limitiations noted above).
+
+In general terms, to use the experimental NVIDIA Container Runtime to launch a
+container with GPU support, it should be inserted as a shim for the desired
+low-level OCI-compliant runtime (e.g. `runc` or `crun`). How this is achieved
+depends on how containers are being launched.
+
+### Docker
+In the case of `docker` for example, the runtime must be registered with the
+Docker daemon. This can be done by modifying the `/etc/docker/daemon.json` file
+to contain the following:
+```json
+{
+    "runtimes": {
+        "nvidia-experimental": {
+                "path": "nvidia-container-runtime.experimental",
+                "runtimeArgs": []
+        }
+    },
+}
+```
+This can then be invoked from docker by including the `--runtime=nvidia-experimental`
+option when executing a `docker run` command.
+
+### Runc
+
+If `runc` is being used to run a container directly substituting the `runc`
+command for `nvidia-container-runtime.experimental` should be sufficient as
+the latter will `exec` to `runc` once the required (in-place) modifications have
+been made to the container's OCI spec (`config.json` file).
+
+## Configuration
+
+### Runtime Path
+The experimental NVIDIA Container Runtime allows for the path to the low-level
+runtime to be specified. This is done by setting the following option in the
+`/etc/nvidia-container-runtime/config.toml` file or setting the
+`NVIDIA_CONTAINER_RUNTIME_PATH` environment variable.
+
+```toml
+[nvidia-container-runtime.experimental]
+
+runtime-path = "/path/to/low-level-runtime"
+```
+This path can be set to the path for `runc` or `crun` on a system and if it is
+a relative path, the `PATH` is searched for a matching executable.
+
+### Device Selection
+In order to select a specific device, the experimental NVIDIA Container Runtime
+mimics the behaviour of the "classic" runtime. That is to say that the values of
+certain [environment variables](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/user-guide.html#environment-variables-oci-spec)
+**in the container's OCI specification** control the behaviour of the runtime.
+
+#### `NVIDIA_VISIBLE_DEVICES`
+This variable controls which GPUs will be made accessible inside the container.
+
+##### Possible values
+* `0,1,2`, `GPU-fef8089b` …: a comma-separated list of GPU UUID(s) or index(es).
+* `all`: all GPUs will be accessible, this is the default value in our container images.
+* `none`: no GPU will be accessible, but driver capabilities will be enabled.
+* `void` or *empty* or *unset*: `nvidia-container-runtime` will have the same behavior as `runc`.
+
+**Note**: When running on a MIG capable device, the following values will also be available:
+* `0:0,0:1,1:0`, `MIG-GPU-fef8089b/0/1` …: a comma-separated list of MIG Device UUID(s) or index(es).
+
+Where the MIG device indices have the form `<GPU Device Index>:<MIG Device Index>` as seen in the example output:
+```
+$ nvidia-smi -L
+GPU 0: Graphics Device (UUID: GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5)
+  MIG Device 0: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/1/0)
+  MIG Device 1: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/1/1)
+  MIG Device 2: (UUID: MIG-GPU-b8ea3855-276c-c9cb-b366-c6fa655957c5/11/0)
+```
+
+#### `NVIDIA_MIG_CONFIG_DEVICES`
+This variable controls which of the visible GPUs can have their MIG
+configuration managed from within the container. This includes enabling and
+disabling MIG mode, creating and destroying GPU Instances and Compute
+Instances, etc.
+
+##### Possible values
+* `all`: Allow all MIG-capable GPUs in the visible device list to have their
+  MIG configurations managed.
+
+**Note**:
+* This feature is only available on MIG capable devices (e.g. the A100).
+* To use this feature, the container must be started with `CAP_SYS_ADMIN` privileges.
+* When not running as `root`, the container user must have read access to the
+  `/proc/driver/nvidia/capabilities/mig/config` file on the host.
+
+#### `NVIDIA_MIG_MONITOR_DEVICES`
+This variable controls which of the visible GPUs can have aggregate information
+about all of their MIG devices monitored from within the container. This
+includes inspecting the aggregate memory usage, listing the aggregate running
+processes, etc.
+
+##### Possible values
+* `all`: Allow all MIG-capable GPUs in the visible device list to have their
+  MIG devices monitored.
+
+**Note**:
+* This feature is only available on MIG capable devices (e.g. the A100).
+* To use this feature, the container must be started with `CAP_SYS_ADMIN` privileges.
+* When not running as `root`, the container user must have read access to the
+  `/proc/driver/nvidia/capabilities/mig/monitor` file on the host.
+

cmd/nvidia-container-runtime.experimental/config/composite.go

@@ -0,0 +1,52 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// list represents a list of config updaters that are applied in order, with later
+// configs having preference.
+type list struct {
+	logger  *log.Logger
+	configs []configUpdater
+}
+
+var _ configUpdater = (*list)(nil)
+
+func newListWithLogger(logger *log.Logger, ci ...configUpdater) configUpdater {
+	c := list{
+		logger:  logger,
+		configs: ci,
+	}
+	return &c
+}
+
+func (c list) Update(cfg *Config) error {
+	for i, u := range c.configs {
+		c.logger.Debugf("Applying config %v: %v", i, u)
+		err := u.Update(cfg)
+		if err != nil {
+			return fmt.Errorf("error applying config %v: %v", i, err)
+		}
+	}
+
+	return nil
+}

cmd/nvidia-container-runtime.experimental/config/composite_test.go

@@ -0,0 +1,100 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestComposite(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	// Empty list
+	c := newListWithLogger(logger)
+
+	cfg := &Config{}
+	err := c.Update(cfg)
+
+	require.NoError(t, err)
+	require.EqualValues(t, &Config{}, cfg)
+
+	// Add a single mock config
+	mockConfigUpdater := &configUpdaterMock{
+		UpdateFunc: func(cfg *Config) error {
+			cfg.DebugFilePath = "updated"
+			return nil
+		},
+	}
+
+	c = newListWithLogger(logger, mockConfigUpdater)
+	cfg = &Config{}
+	err = c.Update(cfg)
+	require.NoError(t, err)
+	require.EqualValues(t, &Config{DebugFilePath: "updated"}, cfg)
+
+	require.Len(t, mockConfigUpdater.UpdateCalls(), 1)
+
+	// Reset the calls
+	mockConfigUpdater.calls = struct{ Update []struct{ Config *Config } }{}
+
+	// define an error config
+	errorConfigUpdater := &configUpdaterMock{
+		UpdateFunc: func(cfg *Config) error {
+			return fmt.Errorf("mock error")
+		},
+	}
+
+	c = newListWithLogger(logger, errorConfigUpdater, mockConfigUpdater)
+	cfg = &Config{}
+	err = c.Update(cfg)
+	require.Error(t, err)
+	require.EqualValues(t, &Config{}, cfg)
+
+	require.Len(t, errorConfigUpdater.UpdateCalls(), 1)
+	require.Len(t, mockConfigUpdater.UpdateCalls(), 0)
+
+	// Reset the calls
+	mockConfigUpdater.calls = struct{ Update []struct{ Config *Config } }{}
+	errorConfigUpdater.calls = struct{ Update []struct{ Config *Config } }{}
+
+	// Change the order of the config and error
+	c = newListWithLogger(logger, mockConfigUpdater, errorConfigUpdater)
+	cfg = &Config{}
+	err = c.Update(cfg)
+	require.Error(t, err)
+	require.EqualValues(t, &Config{DebugFilePath: "updated"}, cfg)
+
+	require.Len(t, errorConfigUpdater.UpdateCalls(), 1)
+	require.Len(t, mockConfigUpdater.UpdateCalls(), 1)
+
+	// Reset the calls
+	mockConfigUpdater.calls = struct{ Update []struct{ Config *Config } }{}
+	errorConfigUpdater.calls = struct{ Update []struct{ Config *Config } }{}
+
+	// Call the mock twice
+	c = newListWithLogger(logger, mockConfigUpdater, mockConfigUpdater)
+	cfg = &Config{}
+	err = c.Update(cfg)
+	require.NoError(t, err)
+	require.EqualValues(t, &Config{DebugFilePath: "updated"}, cfg)
+
+	require.Len(t, mockConfigUpdater.UpdateCalls(), 2)
+}

cmd/nvidia-container-runtime.experimental/config/config.go

@@ -0,0 +1,63 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// Config defines the configuration options for the NVIDIA Container Runtime
+type Config struct {
+	// Root defines the root of the file system to be used for locating mounts
+	Root string
+	// DebugFilePath defines a log file to print debug output to
+	DebugFilePath string
+	// RuntimePath defines the path to an OCI compliant runtime
+	RuntimePath string
+	// LogLevel defines the logging level for the application
+	LogLevel string
+}
+
+//go:generate moq -stub -out config_mock.go . configUpdater
+
+// configUpdate represents an interface for applying updates to a config.
+type configUpdater interface {
+	Update(*Config) error
+}
+
+// GetConfig returns a config struct with the values resolved. The values are defined in order of
+// priority:
+// 1. From the associated environment variables
+// 2. From the loaded config file
+// 3. From the default values defined in the `defaultConfig` function
+func GetConfig(logger *log.Logger) (*Config, error) {
+	cfg := &Config{}
+
+	configs := newListWithLogger(logger,
+		newDefaultConfig(),
+		newDefaultConfigFileWithLogger(logger),
+		newConfigFromEnvironment(),
+	)
+
+	err := configs.Update(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("error getting config: %v", err)
+	}
+	return cfg, nil
+}

cmd/nvidia-container-runtime.experimental/config/config_mock.go

@@ -0,0 +1,76 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package config
+
+import (
+	"sync"
+)
+
+// Ensure, that configUpdaterMock does implement configUpdater.
+// If this is not the case, regenerate this file with moq.
+var _ configUpdater = &configUpdaterMock{}
+
+// configUpdaterMock is a mock implementation of configUpdater.
+//
+// 	func TestSomethingThatUsesconfigUpdater(t *testing.T) {
+//
+// 		// make and configure a mocked configUpdater
+// 		mockedconfigUpdater := &configUpdaterMock{
+// 			UpdateFunc: func(config *Config) error {
+// 				panic("mock out the Update method")
+// 			},
+// 		}
+//
+// 		// use mockedconfigUpdater in code that requires configUpdater
+// 		// and then make assertions.
+//
+// 	}
+type configUpdaterMock struct {
+	// UpdateFunc mocks the Update method.
+	UpdateFunc func(config *Config) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Update holds details about calls to the Update method.
+		Update []struct {
+			// Config is the config argument value.
+			Config *Config
+		}
+	}
+	lockUpdate sync.RWMutex
+}
+
+// Update calls UpdateFunc.
+func (mock *configUpdaterMock) Update(config *Config) error {
+	callInfo := struct {
+		Config *Config
+	}{
+		Config: config,
+	}
+	mock.lockUpdate.Lock()
+	mock.calls.Update = append(mock.calls.Update, callInfo)
+	mock.lockUpdate.Unlock()
+	if mock.UpdateFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.UpdateFunc(config)
+}
+
+// UpdateCalls gets all the calls that were made to Update.
+// Check the length with:
+//     len(mockedconfigUpdater.UpdateCalls())
+func (mock *configUpdaterMock) UpdateCalls() []struct {
+	Config *Config
+} {
+	var calls []struct {
+		Config *Config
+	}
+	mock.lockUpdate.RLock()
+	calls = mock.calls.Update
+	mock.lockUpdate.RUnlock()
+	return calls
+}

cmd/nvidia-container-runtime.experimental/config/config_test.go

@@ -0,0 +1,58 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package config
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetConfigWithCustomConfig(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	// By default debug is disabled
+	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
+	testDir := path.Join(wd, "test")
+	filename := path.Join(testDir, configFileRelativePath)
+
+	previousConfig, present := os.LookupEnv(configOverride)
+	os.Setenv(configOverride, testDir)
+	defer func() {
+		if present {
+			os.Setenv(configOverride, previousConfig)
+		} else {
+			os.Unsetenv(configOverride)
+		}
+	}()
+
+	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
+	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
+
+	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
+
+	logger, _ := testlog.NewNullLogger()
+	cfg, err := GetConfig(logger)
+	require.NoError(t, err)
+	require.Equal(t, "/nvidia-container-toolkit.log", cfg.DebugFilePath)
+}

cmd/nvidia-container-runtime.experimental/config/default.go

@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+/*
 # Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -13,13 +12,36 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+*/
 
-# This script is used to build the packages for the components of the NVIDIA
-# Container Stack. These include the nvidia-container-toolkit in this repository
-# as well as the components included in the third_party folder.
-# All required packages are generated in the specified dist folder.
+package config
 
-test_repo=$1
-echo "Setting up TEST repo: ${test_repo}"
-sed -i -e "s#nvidia\.github\.io/libnvidia-container#${test_repo}/libnvidia-container#g" /etc/yum.repos.d/nvidia-container-toolkit.repo
-yum-config-manager --enable libnvidia-container-experimental
+import (
+	log "github.com/sirupsen/logrus"
+)
+
+type defaultConfig struct{}
+
+var _ configUpdater = (*defaultConfig)(nil)
+
+func newDefaultConfig() configUpdater {
+	c := defaultConfig{}
+	return &c
+}
+
+// Update defines the the default values for the config options
+func (c defaultConfig) Update(cfg *Config) error {
+	*cfg = Config{
+		DebugFilePath: "/dev/null",
+		LogLevel:      log.InfoLevel.String(),
+	}
+	return nil
+}
+
+func getDefaultConfig() *Config {
+	cfg := &Config{}
+
+	defaultConfig{}.Update(cfg)
+
+	return cfg
+}

cmd/nvidia-container-runtime.experimental/config/default_test.go

@@ -0,0 +1,37 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package config
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDefaultConfigUpdate(t *testing.T) {
+	cfg := &Config{}
+
+	require.Empty(t, cfg.DebugFilePath)
+
+	c := defaultConfig{}
+
+	err := c.Update(cfg)
+
+	require.NoError(t, err)
+	require.Equal(t, "/dev/null", cfg.DebugFilePath)
+	require.Equal(t, "", cfg.Root)
+}

cmd/nvidia-container-runtime.experimental/config/environment.go

@@ -0,0 +1,61 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package config
+
+import (
+	"os"
+	"strings"
+)
+
+const (
+	debugFilePathEnvvarName = "NVIDIA_CONTAINER_RUNTIME_DEBUG"
+	runtimePathEnvvarName   = "NVIDIA_CONTAINER_RUNTIME_PATH"
+	rootEnvvarName          = "NVIDIA_CONTAINER_RUNTIME_ROOT"
+	logLevelEnvvarName      = "NVIDIA_CONTAINER_RUNTIME_LOG_LEVEL"
+)
+
+type envConfig struct{}
+
+func newConfigFromEnvironment() configUpdater {
+	c := envConfig{}
+
+	return &c
+}
+
+func (c envConfig) Update(cfg *Config) error {
+	debugFilePathEnvvar, exists := os.LookupEnv(debugFilePathEnvvarName)
+	if exists && strings.TrimSpace(debugFilePathEnvvar) != "" {
+		cfg.DebugFilePath = debugFilePathEnvvar
+	}
+
+	runtimePathEnvvar, exists := os.LookupEnv(runtimePathEnvvarName)
+	if exists && strings.TrimSpace(runtimePathEnvvar) != "" {
+		cfg.RuntimePath = runtimePathEnvvar
+	}
+
+	rootEnvvar, exists := os.LookupEnv(rootEnvvarName)
+	if exists && strings.TrimSpace(rootEnvvar) != "" {
+		cfg.Root = rootEnvvar
+	}
+
+	logLevelEnvvar, exists := os.LookupEnv(logLevelEnvvarName)
+	if exists && strings.TrimSpace(logLevelEnvvar) != "" {
+		cfg.LogLevel = logLevelEnvvar
+	}
+
+	return nil
+}

cmd/nvidia-container-runtime.experimental/config/noop.go

@@ -1,5 +1,5 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,23 +12,18 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-**/
+*/
 
-package main
+package config
 
-import (
-	"os"
+// noop implements a configUpdater that does not update a config.
+type noop struct{}
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
-)
-
-func main() {
-	rt := runtime.New(
-		runtime.WithModeOverride("legacy"),
-	)
-
-	err := rt.Run(os.Args)
-	if err != nil {
-		os.Exit(1)
-	}
+func newNoopConfigUpdater() configUpdater {
+	c := noop{}
+	return &c
+}
+
+func (c noop) Update(cfg *Config) error {
+	return nil
 }

cmd/nvidia-container-runtime.experimental/config/noop_test.go

@@ -1,5 +1,5 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,23 +12,25 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-**/
+*/
 
-package runtime
+package config
 
 import (
 	"testing"
 
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
-func TestLogger(t *testing.T) {
-	l := NewLogger()
+func TestNoopDoesNotModifyConfig(t *testing.T) {
+	cfg := &Config{
+		DebugFilePath: "test-path",
+	}
 
-	l.Update("", "debug", nil)
+	c := newNoopConfigUpdater()
 
-	require.Equal(t, logrus.DebugLevel, l.Logger.Level)
-	require.Equal(t, logrus.InfoLevel, l.previousLogger.Level)
+	err := c.Update(cfg)
 
+	require.NoError(t, err)
+	require.Equal(t, "test-path", cfg.DebugFilePath)
 }

cmd/nvidia-container-runtime.experimental/config/toml.go

@@ -0,0 +1,158 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	configFileRelativePath = "nvidia-container-runtime/config.toml"
+	configOverride         = "XDG_CONFIG_HOME"
+	defaultConfigRoot      = "/etc"
+
+	nvidiaContainerCliSection                       = "nvidia-container-cli"
+	nvidiaContainerRuntimeConfigSection             = "nvidia-container-runtime"
+	nvidiaContainerRuntimeExperimentalConfigSection = "nvidia-container-runtime.experimental"
+)
+
+type tomlConfig struct {
+	logger   *log.Logger
+	path     string
+	sections []tomlSection
+}
+
+type tomlSection struct {
+	section string
+	keys    map[string]struct{}
+}
+
+func newDefaultConfigFileWithLogger(logger *log.Logger) configUpdater {
+	configDir := defaultConfigRoot
+	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
+		configDir = XDGConfigDir
+	}
+	configFilePath := filepath.Join(configDir, configFileRelativePath)
+
+	return newConfigFromFileWithLogger(logger, configFilePath)
+}
+
+func newConfigFromFileWithLogger(logger *log.Logger, filepath string) configUpdater {
+	if _, err := os.Stat(filepath); os.IsNotExist(err) {
+		logger.Warnf("The config file '%v' does not exist", filepath)
+		return newNoopConfigUpdater()
+	}
+
+	sections := []tomlSection{
+		{
+			section: nvidiaContainerRuntimeConfigSection,
+		},
+		{
+			section: nvidiaContainerRuntimeExperimentalConfigSection,
+		},
+		{
+			section: nvidiaContainerCliSection,
+			keys: map[string]struct{}{
+				"root": {},
+			},
+		},
+	}
+
+	c := tomlConfig{
+		logger:   logger,
+		path:     filepath,
+		sections: sections,
+	}
+
+	return &c
+}
+
+func (c tomlConfig) Update(cfg *Config) error {
+	configFile, err := os.Open(c.path)
+	if err != nil {
+		return fmt.Errorf("error opening config file %v: %v", c.path, err)
+	}
+	defer configFile.Close()
+
+	return c.updateFromReader(cfg, configFile)
+}
+
+func (c tomlConfig) updateFromReader(cfg *Config, reader io.Reader) error {
+	toml, err := toml.LoadReader(reader)
+	if err != nil {
+		return fmt.Errorf("error reading TOML contents: %v", err)
+	}
+
+	for _, section := range c.sections {
+		if v, ok := section.GetStringFrom(toml, "debug"); ok {
+			cfg.DebugFilePath = v
+		}
+
+		if v, ok := section.GetStringFrom(toml, "runtime-path"); ok {
+			cfg.RuntimePath = v
+		}
+
+		if v, ok := section.GetStringFrom(toml, "root"); ok {
+			cfg.Root = v
+		}
+
+		if v, ok := section.GetStringFrom(toml, "log-level"); ok {
+			cfg.LogLevel = v
+		}
+	}
+	return nil
+}
+
+func (c tomlSection) GetStringFrom(toml *toml.Tree, key string) (string, bool) {
+	value := c.GetFrom(toml, key)
+	if value != nil {
+		if v, ok := value.(string); ok {
+			return v, ok
+		}
+	}
+	return "", false
+}
+
+func (c tomlSection) GetFrom(toml *toml.Tree, key string) interface{} {
+	if !c.validKey(key) {
+		return nil
+	}
+	return toml.Get(c.configKey(key))
+}
+
+func (c tomlSection) validKey(key string) bool {
+	if c.keys == nil {
+		return true
+	}
+
+	_, exists := c.keys[key]
+
+	return exists
+}
+
+func (c tomlSection) configKey(key string) string {
+	if c.section == "" {
+		return key
+	}
+	return c.section + "." + key
+}

cmd/nvidia-container-runtime.experimental/config/toml_test.go

@@ -0,0 +1,259 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package config
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
+	"strings"
+	"testing"
+	"testing/iotest"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateFromReader(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		description   string
+		readerError   bool
+		lines         []string
+		expected      *Config
+		expectedError bool
+	}{
+		{
+			description:   "reader error returns error",
+			readerError:   true,
+			expectedError: true,
+			expected:      getDefaultConfig(),
+		},
+		{
+			description: "empty config returns defaults",
+			lines:       []string{},
+			expected: &Config{
+				DebugFilePath: "/dev/null",
+				LogLevel:      "info",
+			},
+		},
+		{
+			description: "debug output is set",
+			lines:       []string{"nvidia-container-runtime.debug=\"nvidia-container-toolkit.log\""},
+			expected: &Config{
+				DebugFilePath: "nvidia-container-toolkit.log",
+				LogLevel:      "info",
+			},
+		},
+		{
+			description: "debug output is set in section",
+			lines:       []string{"[nvidia-container-runtime]", "debug=\"nvidia-container-toolkit.log\""},
+			expected: &Config{
+				DebugFilePath: "nvidia-container-toolkit.log",
+				LogLevel:      "info",
+			},
+		},
+		{
+			description: "blank debug is set",
+			lines:       []string{"nvidia-container-runtime.debug=\"\""},
+			expected: &Config{
+				DebugFilePath: "",
+				LogLevel:      "info",
+			},
+		},
+		{
+			description: "non-string debug is ignored",
+			lines:       []string{"nvidia-container-runtime.debug=2"},
+			expected: &Config{
+				DebugFilePath: "/dev/null",
+				LogLevel:      "info",
+			},
+		},
+		{
+			description: "log-level is set",
+			lines:       []string{"nvidia-container-runtime.log-level=\"trace\""},
+			expected: &Config{
+				DebugFilePath: "/dev/null",
+				LogLevel:      "trace",
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			cfg := getDefaultConfig()
+
+			c := tomlConfig{
+				logger: logger,
+				sections: []tomlSection{
+					{section: nvidiaContainerRuntimeConfigSection},
+				},
+			}
+			var reader io.Reader
+			if tc.readerError {
+				reader = iotest.ErrReader(fmt.Errorf("error"))
+			} else {
+				reader = strings.NewReader(strings.Join(tc.lines, "\n"))
+			}
+
+			err := c.updateFromReader(cfg, reader)
+
+			if tc.expectedError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+			}
+			require.EqualValues(t, tc.expected, cfg)
+		})
+	}
+}
+
+func TestUpdateFromReaderExperimental(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		readerError   bool
+		lines         []string
+		expected      *Config
+		expectedError bool
+	}{
+		{
+			lines:    []string{"nvidia-container-runtime.debug=\"nvidia-container-toolkit.log\""},
+			expected: getDefaultConfig(),
+		},
+		{
+			lines:    []string{"[nvidia-container-runtime]", "debug=\"nvidia-container-toolkit.log\""},
+			expected: getDefaultConfig(),
+		},
+		{
+			lines: []string{"nvidia-container-runtime.experimental.debug=\"\""},
+			expected: &Config{
+				DebugFilePath: "",
+				LogLevel:      "info",
+			},
+		},
+		{
+			lines:    []string{"nvidia-container-runtime.experimental.debug=2"},
+			expected: getDefaultConfig(),
+		},
+		{
+			lines: []string{"nvidia-container-runtime.experimental.debug=\"nvidia-container-toolkit.log\""},
+			expected: &Config{
+				DebugFilePath: "nvidia-container-toolkit.log",
+				LogLevel:      "info",
+			},
+		},
+		{
+			lines: []string{"[nvidia-container-runtime.experimental]", "debug=\"nvidia-container-toolkit.log\""},
+			expected: &Config{
+				DebugFilePath: "nvidia-container-toolkit.log",
+				LogLevel:      "info",
+			},
+		},
+		{
+			lines: []string{
+				"nvidia-container-runtime.debug=\"nvidia-container-toolkit.log\"",
+				"nvidia-container-runtime.experimental.debug=\"nvidia-container-exp-toolkit.log\"",
+			},
+			expected: &Config{
+				DebugFilePath: "nvidia-container-exp-toolkit.log",
+				LogLevel:      "info",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		cfg := getDefaultConfig()
+
+		c := tomlConfig{
+			logger: logger,
+			sections: []tomlSection{
+				{section: nvidiaContainerRuntimeExperimentalConfigSection},
+			},
+		}
+		var reader io.Reader
+		if tc.readerError {
+			reader = iotest.ErrReader(fmt.Errorf("error"))
+		} else {
+			reader = strings.NewReader(strings.Join(tc.lines, "\n"))
+		}
+
+		err := c.updateFromReader(cfg, reader)
+
+		if tc.expectedError {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+		require.EqualValues(t, tc.expected, cfg, "%d: %v", i, tc)
+	}
+}
+
+func TestConfigFromFile(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	// By default debug is disabled
+	lines := []string{
+		"[nvidia-container-cli]",
+		"root = \"/run/nvidia/driver\"",
+		"[nvidia-container-runtime]",
+		"#debug = \"/nvidia-container-toolkit.log\"",
+		"",
+		"[nvidia-container-runtime.experimental]",
+		"debug = \"/nvidia-container-toolkit.experimental.log\"",
+	}
+
+	contents := []byte(strings.Join(lines, "\n"))
+	testDir := path.Join(wd, "test")
+	filename := path.Join(testDir, configFileRelativePath)
+
+	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
+	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
+	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
+
+	logger, _ := testlog.NewNullLogger()
+	c := newConfigFromFileWithLogger(logger, filename)
+
+	cfg := getDefaultConfig()
+
+	err = c.Update(cfg)
+
+	require.NoError(t, err)
+	require.Equal(t, "/nvidia-container-toolkit.experimental.log", cfg.DebugFilePath)
+
+	require.Equal(t, "/run/nvidia/driver", cfg.Root)
+}
+
+func TestConfigFromNonexistentFileReturnsNoop(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+	c := newConfigFromFileWithLogger(logger, "/does/not/exist")
+
+	n, ok := c.(*noop)
+
+	require.True(t, ok)
+	require.NotNil(t, n)
+}
+
+func TestGetConfigKey(t *testing.T) {
+	require.Equal(t, "key", tomlSection{}.configKey("key"))
+	require.Equal(t, "section.key", tomlSection{section: "section"}.configKey("key"))
+}

cmd/nvidia-container-runtime.experimental/main.go

@@ -0,0 +1,144 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ensure"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/filter"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/modify"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental/config"
+)
+
+const (
+	visibleDevicesEnvvar = "NVIDIA_VISIBLE_DEVICES"
+	visibleDevicesVoid   = "void"
+)
+
+var logger = log.New()
+
+func main() {
+	cfg, err := config.GetConfig(logger)
+	if err != nil {
+		logger.Errorf("Error loading config: %v", err)
+		os.Exit(1)
+	}
+
+	if cfg.DebugFilePath != "" && cfg.DebugFilePath != "/dev/null" {
+		logFile, err := os.OpenFile(cfg.DebugFilePath, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+		if err != nil {
+			logger.Errorf("Error opening debug log file: %v", err)
+			os.Exit(1)
+		}
+		defer logFile.Close()
+
+		logger.SetOutput(logFile)
+	}
+
+	logLevel, err := log.ParseLevel(cfg.LogLevel)
+	if err == nil {
+		logger.SetLevel(logLevel)
+	} else {
+		logger.Warnf("Invalid log-level '%v'; using '%v'", cfg.LogLevel, logger.Level.String())
+	}
+
+	logger.Infof("Starting nvidia-container-runtime: %v", os.Args)
+	logger.Debugf("Using config=%+v", cfg)
+
+	if err := run(cfg, os.Args); err != nil {
+		logger.Errorf("Error running runtime: %v", err)
+		os.Exit(1)
+	}
+}
+
+func run(cfg *config.Config, args []string) error {
+	logger.Debugf("running with args=%v", args)
+
+	// We create a low-level runtime
+	lowLevelRuntime, err := createLowLevelRuntime(logger, cfg)
+	if err != nil {
+		return fmt.Errorf("error constructing low-level runtime: %v", err)
+	}
+
+	if !oci.HasCreateSubcommand(args) {
+		logger.Infof("No modification of OCI specification required")
+		logger.Infof("Forwarding command to runtime")
+		return lowLevelRuntime.Exec(args)
+	}
+
+	// We create the OCI spec that is to be modified
+	ociSpec, bundleDir, err := oci.NewSpecFromArgs(args)
+	if err != nil {
+		return fmt.Errorf("error constructing OCI spec: %v", err)
+	}
+
+	err = ociSpec.Load()
+	if err != nil {
+		return fmt.Errorf("error loading OCI specification: %v", err)
+	}
+
+	visibleDevices, exists := ociSpec.LookupEnv(visibleDevicesEnvvar)
+	if !exists || visibleDevices == "" || visibleDevices == visibleDevicesVoid {
+		logger.Infof("Using low-level runtime: %v=%v (exists=%v)", visibleDevicesEnvvar, visibleDevices, exists)
+		return lowLevelRuntime.Exec(os.Args)
+	}
+
+	// We create the modifier that will be applied by the Modifying Runtime Wrapper
+	modifier, err := createModifier(cfg.Root, bundleDir, visibleDevices, ociSpec)
+	if err != nil {
+		return fmt.Errorf("error constructing modifer: %v", err)
+	}
+
+	// We construct the Modifying runtime
+	r := runtime.NewModifyingRuntimeWrapperWithLogger(logger, lowLevelRuntime, ociSpec, modifier)
+
+	return r.Exec(os.Args)
+}
+
+func createLowLevelRuntime(logger *log.Logger, cfg *config.Config) (oci.Runtime, error) {
+	if cfg.RuntimePath == "" {
+		return oci.NewLowLevelRuntimeWithLogger(logger, "docker-runc", "runc")
+	}
+	logger.Infof("Creating runtime with path %v", cfg.RuntimePath)
+	return oci.NewRuntimeForPathWithLogger(logger, cfg.RuntimePath)
+}
+
+func createModifier(root string, bundleDir string, visibleDevices string, env filter.EnvLookup) (modify.Modifier, error) {
+	// We set up the modifier using discovery
+	discovered, err := discover.NewNVMLServerWithLogger(logger, root)
+	if err != nil {
+		return nil, fmt.Errorf("error discovering devices: %v", err)
+	}
+
+	// We apply a filter to the discovered devices
+	selected := filter.NewSelectDevicesFromWithLogger(logger, discovered, visibleDevices, env)
+
+	// We ensure that the selected devices are available
+	available := ensure.NewEnsureDevicesWithLogger(logger, selected, root)
+
+	// We construct the modifer for the OCI spec
+	modifier := modify.NewModifierWithLoggerFor(logger, available, root, bundleDir)
+
+	return modifier, nil
+}

cmd/nvidia-container-runtime/README.md

@@ -1,87 +0,0 @@
-# The NVIDIA Container Runtime
-
-The NVIDIA Container Runtime is a shim for OCI-compliant low-level runtimes such as [runc](https://github.com/opencontainers/runc). When a `create` command is detected, the incoming [OCI runtime specification](https://github.com/opencontainers/runtime-spec) is modified in place and the command is forwarded to the low-level runtime.
-
-## Configuration
-
-The NVIDIA Container Runtime uses file-based configuration, with the config stored in `/etc/nvidia-container-runtime/config.toml`. The `/etc` path can be overridden using the `XDG_CONFIG_HOME` environment variable with the `${XDG_CONFIG_HOME}/nvidia-container-runtime/config.toml` file used instead if this environment variable is set.
-
-This config file may contain options for other components of the NVIDIA container stack and for the NVIDIA Container Runtime, the relevant config section is `nvidia-container-runtime`
-
-### Logging
-
-The `log-level` config option (default: `"info"`) specifies the log level to use and the `debug` option, if set, specifies a log file to which logs for the NVIDIA Container Runtime must be written.
-
-In addition to this, the NVIDIA Container Runtime considers the value of `--log` and `--log-format` flags that may be passed to it by a container runtime such as docker or containerd. If the `--debug` flag is present the log-level specified in the config file is overridden as `"debug"`.
-
-### Low-level Runtime Path
-
-The `runtimes` config option allows for the low-level runtime to be specified. The first entry in this list that is an existing executable file is used as the low-level runtime. If the entry is not a path, the `PATH` is searched for a matching executable. If the entry is a path this is checked instead.
-
-The default value for this setting is:
-```toml
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-```
-
-and if, for example, `crun` is to be used instead this can be changed to:
-```toml
-runtimes = [
-    "crun",
-]
-```
-
-### Runtime Mode
-
-The `mode` config option (default `"auto"`) controls the high-level behaviour of the runtime.
-
-#### Auto Mode
-
-When `mode` is set to `"auto"`, the runtime employs heuristics to determine which mode to use based on, for example, the platform where the runtime is being run.
-
-#### Legacy Mode
-
-When `mode` is set to `"legacy"`, the NVIDIA Container Runtime adds a [`prestart` hook](https://github.com/opencontainers/runtime-spec/blob/master/config.md#prestart) to the incomming OCI specification that invokes the NVIDIA Container Runtime Hook for all containers created. This hook checks whether NVIDIA devices are requested and ensures GPU access is configured using the `nvidia-container-cli` from the [libnvidia-container](https://github.com/NVIDIA/libnvidia-container) project.
-
-#### CSV Mode
-
-When `mode` is set to `"csv"`, CSV files at `/etc/nvidia-container-runtime/host-files-for-container.d` define the devices and mounts that are to be injected into a container when it is created. The search path for the files can be overridden by modifying the `nvidia-container-runtime.modes.csv.mount-spec-path` in the config as below:
-
-```toml
-[nvidia-container-runtime]
-    [nvidia-container-runtime.modes.csv]
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"
-```
-
-This mode is primarily targeted at Tegra-based systems without NVML available.
-
-### Notes on using the docker CLI
-
-Note that only the `"legacy"` NVIDIA Container Runtime mode is directly compatible with the `--gpus` flag implemented by the `docker` CLI (assuming the NVIDIA Container Runtime is not used). The reason for this is that `docker` inserts the same NVIDIA Container Runtime Hook into the OCI runtime specification.
-
-
-If a different mode is explicitly set or detected, the NVIDIA Container Runtime Hook will raise the following error when `--gpus` is set:
-```
-$ docker run --rm --gpus all ubuntu:18.04
-docker: Error response from daemon: failed to create shim: OCI runtime create failed: container_linux.go:380: starting container process caused: process_linux.go:545: container init caused: Running hook #0:: error running hook: exit status 1, stdout: , stderr: Auto-detected mode as 'csv'
-invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime instead.: unknown.
-```
-Here NVIDIA Container Runtime must be used explicitly. The recommended way to do this is to specify the `--runtime=nvidia` command line argument as part of the `docker run` commmand as follows:
-```
-$ docker run --rm --gpus all --runtime=nvidia ubuntu:18.04
-```
-
-Alternatively the NVIDIA Container Runtime can be set as the default runtime for docker. This can be done by modifying the `/etc/docker/daemon.json` file as follows:
-```json
-{
-    "default-runtime": "nvidia",
-    "runtimes": {
-        "nvidia": {
-            "path": "nvidia-container-runtime",
-            "runtimeArgs": []
-        }
-    }
-}
-```

cmd/nvidia-container-runtime/logger.go

@@ -0,0 +1,79 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/tsaikd/KDGoLib/logrusutil"
+)
+
+// Logger adds a way to manage output to a log file to a logrus.Logger
+type Logger struct {
+	*logrus.Logger
+	previousOutput io.Writer
+	logFile        *os.File
+}
+
+// NewLogger constructs a Logger with a preddefined formatter
+func NewLogger() *Logger {
+	logrusLogger := logrus.New()
+
+	formatter := &logrusutil.ConsoleLogFormatter{
+		TimestampFormat: "2006/01/02 15:04:07",
+		Flag:            logrusutil.Ltime,
+	}
+
+	logger := &Logger{
+		Logger: logrusLogger,
+	}
+	logger.SetFormatter(formatter)
+
+	return logger
+}
+
+// LogToFile opens the specified file for appending and sets the logger to
+// output to the opened file. A reference to the file pointer is stored to
+// allow this to be closed.
+func (l *Logger) LogToFile(filename string) error {
+	logFile, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("error opening debug log file: %v", err)
+	}
+
+	l.logFile = logFile
+	l.previousOutput = l.Out
+	l.SetOutput(logFile)
+
+	return nil
+}
+
+// CloseFile closes the log file (if any) and resets the logger output to what it
+// was before LogToFile was called.
+func (l *Logger) CloseFile() error {
+	if l.logFile == nil {
+		return nil
+	}
+	logFile := l.logFile
+	l.SetOutput(l.previousOutput)
+	l.logFile = nil
+
+	return logFile.Close()
+}

cmd/nvidia-container-runtime/main.go

@@ -1,15 +1,89 @@
 package main
 
 import (
+	"fmt"
 	"os"
+	"path"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/runtime"
+	"github.com/pelletier/go-toml"
 )
 
+const (
+	configOverride = "XDG_CONFIG_HOME"
+	configFilePath = "nvidia-container-runtime/config.toml"
+
+	hookDefaultFilePath = "/usr/bin/nvidia-container-runtime-hook"
+)
+
+var (
+	configDir = "/etc/"
+)
+
+var logger = NewLogger()
+
 func main() {
-	r := runtime.New()
-	err := r.Run(os.Args)
+	err := run(os.Args)
 	if err != nil {
+		logger.Errorf("Error running %v: %v", os.Args, err)
 		os.Exit(1)
 	}
 }
+
+// run is an entry point that allows for idiomatic handling of errors
+// when calling from the main function.
+func run(argv []string) (err error) {
+	cfg, err := getConfig()
+	if err != nil {
+		return fmt.Errorf("error loading config: %v", err)
+	}
+
+	err = logger.LogToFile(cfg.debugFilePath)
+	if err != nil {
+		return fmt.Errorf("error opening debug log file: %v", err)
+	}
+	defer func() {
+		// We capture and log a returning error before closing the log file.
+		if err != nil {
+			logger.Errorf("Error running %v: %v", argv, err)
+		}
+		logger.CloseFile()
+	}()
+
+	r, err := newRuntime(argv)
+	if err != nil {
+		return fmt.Errorf("error creating runtime: %v", err)
+	}
+
+	logger.Printf("Running %s\n", argv[0])
+	return r.Exec(argv)
+}
+
+type config struct {
+	debugFilePath string
+}
+
+// getConfig sets up the config struct. Values are read from a toml file
+// or set via the environment.
+func getConfig() (*config, error) {
+	cfg := &config{}
+
+	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
+		configDir = XDGConfigDir
+	}
+
+	configFilePath := path.Join(configDir, configFilePath)
+
+	tomlContent, err := os.ReadFile(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	toml, err := toml.Load(string(tomlContent))
+	if err != nil {
+		return nil, err
+	}
+
+	cfg.debugFilePath = toml.GetDefault("nvidia-container-runtime.debug", "/dev/null").(string)
+
+	return cfg, nil
+}

cmd/nvidia-container-runtime/main_test.go

@@ -3,17 +3,16 @@ package main
 import (
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strings"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/modifier"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/test"
 	"github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/require"
 )
 
@@ -25,10 +24,6 @@ const (
 	unmodifiedSpecFileSuffix = "test/input/test_spec.json"
 )
 
-const (
-	runcExecutableName = "runc"
-)
-
 type testConfig struct {
 	root    string
 	binPath string
@@ -40,25 +35,25 @@ func TestMain(m *testing.M) {
 	// TEST SETUP
 	// Determine the module root and the test binary path
 	var err error
-	moduleRoot, err := test.GetModuleRoot()
+	moduleRoot, err := getModuleRoot()
 	if err != nil {
-		logrus.Fatalf("error in test setup: could not get module root: %v", err)
+		logger.Fatalf("error in test setup: could not get module root: %v", err)
 	}
 	testBinPath := filepath.Join(moduleRoot, "test", "bin")
 	testInputPath := filepath.Join(moduleRoot, "test", "input")
 
 	// Set the environment variables for the test
-	os.Setenv("PATH", test.PrependToPath(testBinPath, moduleRoot))
+	os.Setenv("PATH", prependToPath(testBinPath, moduleRoot))
 	os.Setenv("XDG_CONFIG_HOME", testInputPath)
 
 	// Confirm that the environment is configured correctly
 	runcPath, err := exec.LookPath(runcExecutableName)
 	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
-		logrus.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
 	}
 	hookPath, err := exec.LookPath(nvidiaHook)
 	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
-		logrus.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
 	}
 
 	// Store the root and binary paths in the test Config
@@ -76,15 +71,45 @@ func TestMain(m *testing.M) {
 	os.Exit(exitCode)
 }
 
+func getModuleRoot() (string, error) {
+	_, filename, _, _ := runtime.Caller(0)
+
+	return hasGoMod(filename)
+}
+
+func hasGoMod(dir string) (string, error) {
+	if dir == "" || dir == "/" {
+		return "", fmt.Errorf("module root not found")
+	}
+
+	_, err := os.Stat(filepath.Join(dir, "go.mod"))
+	if err != nil {
+		return hasGoMod(filepath.Dir(dir))
+	}
+	return dir, nil
+}
+
+func prependToPath(additionalPaths ...string) string {
+	paths := strings.Split(os.Getenv("PATH"), ":")
+	paths = append(additionalPaths, paths...)
+
+	return strings.Join(paths, ":")
+}
+
 // case 1) nvidia-container-runtime run --bundle
 // case 2) nvidia-container-runtime create --bundle
-//   - Confirm the runtime handles bad input correctly
+//		- Confirm the runtime handles bad input correctly
 func TestBadInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
 		t.Fatal(err)
 	}
 
+	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle")
+	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
+	output, err := cmdRun.CombinedOutput()
+	require.Errorf(t, err, "runtime should return an error", "output=%v", string(output))
+
 	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle")
 	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
 	err = cmdCreate.Run()
@@ -92,10 +117,9 @@ func TestBadInput(t *testing.T) {
 }
 
 // case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
-//   - Confirm the runtime runs with no errors
-//
+//		- Confirm the runtime runs with no errors
 // case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
-//   - Confirm the runtime inserts the NVIDIA prestart hook correctly
+//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
 func TestGoodInput(t *testing.T) {
 	err := cfg.generateNewRuntimeSpec()
 	if err != nil {
@@ -169,11 +193,11 @@ func TestDuplicateHook(t *testing.T) {
 	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
 }
 
-// addNVIDIAHook is a basic wrapper for an addHookModifier that is used for
+// addNVIDIAHook is a basic wrapper for nvidiaContainerRunime.addNVIDIAHook that is used for
 // testing.
 func addNVIDIAHook(spec *specs.Spec) error {
-	m := modifier.NewStableRuntimeModifier(logrus.StandardLogger())
-	return m.Modify(spec)
+	r := nvidiaContainerRuntime{logger: logger.Logger}
+	return r.addNVIDIAHook(spec)
 }
 
 func (c testConfig) getRuntimeSpec() (specs.Spec, error) {
@@ -246,3 +270,24 @@ func nvidiaHookCount(hooks *specs.Hooks) int {
 	}
 	return count
 }
+
+func TestGetConfigWithCustomConfig(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	// By default debug is disabled
+	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
+	testDir := filepath.Join(wd, "test")
+	filename := filepath.Join(testDir, configFilePath)
+
+	os.Setenv(configOverride, testDir)
+
+	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
+	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
+
+	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
+
+	cfg, err := getConfig()
+	require.NoError(t, err)
+	require.Equal(t, cfg.debugFilePath, "/nvidia-container-toolkit.log")
+}

cmd/nvidia-container-runtime/nvcr.go

@@ -0,0 +1,132 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	log "github.com/sirupsen/logrus"
+)
+
+// nvidiaContainerRuntime encapsulates the NVIDIA Container Runtime. It wraps the specified runtime, conditionally
+// modifying the specified OCI specification before invoking the runtime.
+type nvidiaContainerRuntime struct {
+	logger  *log.Logger
+	runtime oci.Runtime
+	ociSpec oci.Spec
+}
+
+var _ oci.Runtime = (*nvidiaContainerRuntime)(nil)
+
+// newNvidiaContainerRuntime is a constructor for a standard runtime shim.
+func newNvidiaContainerRuntimeWithLogger(logger *log.Logger, runtime oci.Runtime, ociSpec oci.Spec) (oci.Runtime, error) {
+	r := nvidiaContainerRuntime{
+		logger:  logger,
+		runtime: runtime,
+		ociSpec: ociSpec,
+	}
+
+	return &r, nil
+}
+
+// Exec defines the entrypoint for the NVIDIA Container Runtime. A check is performed to see whether modifications
+// to the OCI spec are required -- and applicable modifcations applied. The supplied arguments are then
+// forwarded to the underlying runtime's Exec method.
+func (r nvidiaContainerRuntime) Exec(args []string) error {
+	if r.modificationRequired(args) {
+		err := r.modifyOCISpec()
+		if err != nil {
+			return fmt.Errorf("error modifying OCI spec: %v", err)
+		}
+	}
+
+	r.logger.Println("Forwarding command to runtime")
+	return r.runtime.Exec(args)
+}
+
+// modificationRequired checks the intput arguments to determine whether a modification
+// to the OCI spec is required.
+func (r nvidiaContainerRuntime) modificationRequired(args []string) bool {
+	if oci.HasCreateSubcommand(args) {
+		r.logger.Infof("'create' command detected; modification required")
+		return true
+	}
+
+	r.logger.Infof("No modification required")
+	return false
+}
+
+// modifyOCISpec loads and modifies the OCI spec specified in the nvidiaContainerRuntime
+// struct. The spec is modified in-place and written to the same file as the input after
+// modifcationas are applied.
+func (r nvidiaContainerRuntime) modifyOCISpec() error {
+	err := r.ociSpec.Load()
+	if err != nil {
+		return fmt.Errorf("error loading OCI specification for modification: %v", err)
+	}
+
+	err = r.ociSpec.Modify(r.addNVIDIAHook)
+	if err != nil {
+		return fmt.Errorf("error injecting NVIDIA Container Runtime hook: %v", err)
+	}
+
+	err = r.ociSpec.Flush()
+	if err != nil {
+		return fmt.Errorf("error writing modified OCI specification: %v", err)
+	}
+	return nil
+}
+
+// addNVIDIAHook modifies the specified OCI specification in-place, inserting a
+// prestart hook.
+func (r nvidiaContainerRuntime) addNVIDIAHook(spec *specs.Spec) error {
+	path, err := exec.LookPath("nvidia-container-runtime-hook")
+	if err != nil {
+		path = hookDefaultFilePath
+		_, err = os.Stat(path)
+		if err != nil {
+			return err
+		}
+	}
+
+	r.logger.Printf("prestart hook path: %s\n", path)
+
+	args := []string{path}
+	if spec.Hooks == nil {
+		spec.Hooks = &specs.Hooks{}
+	} else if len(spec.Hooks.Prestart) != 0 {
+		for _, hook := range spec.Hooks.Prestart {
+			if !strings.Contains(hook.Path, "nvidia-container-runtime-hook") {
+				continue
+			}
+			r.logger.Println("existing nvidia prestart hook in OCI spec file")
+			return nil
+		}
+	}
+
+	spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
+		Path: path,
+		Args: append(args, "prestart"),
+	})
+
+	return nil
+}

cmd/nvidia-container-runtime/nvcr_test.go

@@ -0,0 +1,203 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddNvidiaHook(t *testing.T) {
+	logger, logHook := testlog.NewNullLogger()
+	shim := nvidiaContainerRuntime{
+		logger: logger,
+	}
+
+	testCases := []struct {
+		spec         *specs.Spec
+		errorPrefix  string
+		shouldNotAdd bool
+	}{
+		{
+			spec: &specs.Spec{},
+		},
+		{
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{},
+			},
+		},
+		{
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{{
+						Path: "some-hook",
+					}},
+				},
+			},
+		},
+		{
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{{
+						Path: "nvidia-container-runtime-hook",
+					}},
+				},
+			},
+			shouldNotAdd: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		logHook.Reset()
+
+		var numPrestartHooks int
+		if tc.spec.Hooks != nil {
+			numPrestartHooks = len(tc.spec.Hooks.Prestart)
+		}
+
+		err := shim.addNVIDIAHook(tc.spec)
+
+		if tc.errorPrefix == "" {
+			require.NoErrorf(t, err, "%d: %v", i, tc)
+		} else {
+			require.Truef(t, strings.HasPrefix(err.Error(), tc.errorPrefix), "%d: %v", i, tc)
+
+			require.NotNilf(t, tc.spec.Hooks, "%d: %v", i, tc)
+			require.Equalf(t, 1, nvidiaHookCount(tc.spec.Hooks), "%d: %v", i, tc)
+
+			if tc.shouldNotAdd {
+				require.Equal(t, numPrestartHooks+1, len(tc.spec.Hooks.Poststart), "%d: %v", i, tc)
+			} else {
+				require.Equal(t, numPrestartHooks+1, len(tc.spec.Hooks.Poststart), "%d: %v", i, tc)
+
+				nvidiaHook := tc.spec.Hooks.Poststart[len(tc.spec.Hooks.Poststart)-1]
+
+				// TODO: This assumes that the hook has been set up in the makefile
+				expectedPath := "/usr/bin/nvidia-container-runtime-hook"
+				require.Equalf(t, expectedPath, nvidiaHook.Path, "%d: %v", i, tc)
+				require.Equalf(t, []string{expectedPath, "prestart"}, nvidiaHook.Args, "%d: %v", i, tc)
+				require.Emptyf(t, nvidiaHook.Env, "%d: %v", i, tc)
+				require.Nilf(t, nvidiaHook.Timeout, "%d: %v", i, tc)
+			}
+		}
+	}
+}
+
+func TestNvidiaContainerRuntime(t *testing.T) {
+	logger, hook := testlog.NewNullLogger()
+
+	testCases := []struct {
+		shim         nvidiaContainerRuntime
+		shouldModify bool
+		args         []string
+		modifyError  error
+		writeError   error
+	}{
+		{
+			shim:         nvidiaContainerRuntime{},
+			shouldModify: false,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"--bundle=create"},
+			shouldModify: false,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"--bundle", "create"},
+			shouldModify: false,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"create"},
+			modifyError:  fmt.Errorf("error modifying"),
+			shouldModify: true,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"create"},
+			writeError:   fmt.Errorf("error writing"),
+			shouldModify: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		tc.shim.logger = logger
+		hook.Reset()
+
+		ociMock := &oci.SpecMock{
+			ModifyFunc: func(specModifier oci.SpecModifier) error {
+				return tc.modifyError
+			},
+			FlushFunc: func() error {
+				return tc.writeError
+			},
+		}
+		require.Equal(t, tc.shouldModify, tc.shim.modificationRequired(tc.args), "%d: %v", i, tc)
+
+		tc.shim.ociSpec = ociMock
+		tc.shim.runtime = &MockShim{}
+
+		err := tc.shim.Exec(tc.args)
+		if tc.modifyError != nil || tc.writeError != nil {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+
+		if tc.shouldModify {
+			require.Equal(t, 1, len(ociMock.ModifyCalls()), "%d: %v", i, tc)
+		} else {
+			require.Equal(t, 0, len(ociMock.ModifyCalls()), "%d: %v", i, tc)
+		}
+
+		writeExpected := tc.shouldModify && tc.modifyError == nil
+		if writeExpected {
+			require.Equal(t, 1, len(ociMock.FlushCalls()), "%d: %v", i, tc)
+		} else {
+			require.Equal(t, 0, len(ociMock.FlushCalls()), "%d: %v", i, tc)
+		}
+	}
+}
+
+type MockShim struct {
+	called      bool
+	args        []string
+	returnError error
+}
+
+func (m *MockShim) Exec(args []string) error {
+	m.called = true
+	m.args = args
+	return m.returnError
+}

cmd/nvidia-container-runtime/runtime_factory.go

@@ -0,0 +1,74 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+const (
+	ociSpecFileName          = "config.json"
+	dockerRuncExecutableName = "docker-runc"
+	runcExecutableName       = "runc"
+)
+
+// newRuntime is a factory method that constructs a runtime based on the selected configuration.
+func newRuntime(argv []string) (oci.Runtime, error) {
+	ociSpec, err := newOCISpec(argv)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing OCI specification: %v", err)
+	}
+
+	runc, err := newRuncRuntime()
+	if err != nil {
+		return nil, fmt.Errorf("error constructing runc runtime: %v", err)
+	}
+
+	r, err := newNvidiaContainerRuntimeWithLogger(logger.Logger, runc, ociSpec)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing NVIDIA Container Runtime: %v", err)
+	}
+
+	return r, nil
+}
+
+// newOCISpec constructs an OCI spec for the provided arguments
+func newOCISpec(argv []string) (oci.Spec, error) {
+	bundleDir, err := oci.GetBundleDir(argv)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing command line arguments: %v", err)
+	}
+	logger.Infof("Using bundle directory: %v", bundleDir)
+
+	ociSpecPath := oci.GetSpecFilePath(bundleDir)
+	logger.Infof("Using OCI specification file path: %v", ociSpecPath)
+
+	ociSpec := oci.NewSpecFromFile(ociSpecPath)
+
+	return ociSpec, nil
+}
+
+// newRuncRuntime locates the runc binary and wraps it in a SyscallExecRuntime
+func newRuncRuntime() (oci.Runtime, error) {
+	return oci.NewLowLevelRuntimeWithLogger(
+		logger.Logger,
+		dockerRuncExecutableName,
+		runcExecutableName,
+	)
+}

cmd/nvidia-container-runtime/runtime_factory_test.go

@@ -1,5 +1,5 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -12,20 +12,19 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-**/
+*/
 
-package edits
+package main
 
 import (
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	"github.com/stretchr/testify/require"
 )
 
-func TestFromDiscovererAllowsMountsToIterate(t *testing.T) {
-	edits, err := FromDiscoverer(discover.None{})
-	require.NoError(t, err)
+func TestConstructor(t *testing.T) {
+	shim, err := newRuntime([]string{})
 
-	require.Empty(t, edits.Mounts)
+	require.NoError(t, err)
+	require.NotNil(t, shim)
 }

cmd/nvidia-container-toolkit/container_config.go

@@ -7,13 +7,14 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
-	"github.com/opencontainers/runtime-spec/specs-go"
 	"golang.org/x/mod/semver"
 )
 
+var envSwarmGPU *string
+
 const (
 	envCUDAVersion          = "CUDA_VERSION"
 	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
@@ -103,6 +104,32 @@ type HookState struct {
 	BundlePath string `json:"bundlePath"`
 }
 
+func parseCudaVersion(cudaVersion string) (vmaj, vmin, vpatch uint32) {
+	if _, err := fmt.Sscanf(cudaVersion, "%d.%d.%d\n", &vmaj, &vmin, &vpatch); err != nil {
+		vpatch = 0
+		if _, err := fmt.Sscanf(cudaVersion, "%d.%d\n", &vmaj, &vmin); err != nil {
+			vmin = 0
+			if _, err := fmt.Sscanf(cudaVersion, "%d\n", &vmaj); err != nil {
+				log.Panicln("invalid CUDA version:", cudaVersion)
+			}
+		}
+	}
+
+	return
+}
+
+func getEnvMap(e []string) (m map[string]string) {
+	m = make(map[string]string)
+	for _, s := range e {
+		p := strings.SplitN(s, "=", 2)
+		if len(p) != 2 {
+			log.Panicln("environment error")
+		}
+		m[p[0]] = p[1]
+	}
+	return
+}
+
 func loadSpec(path string) (spec *Spec) {
 	f, err := os.Open(path)
 	if err != nil {
@@ -131,7 +158,7 @@ func isPrivileged(s *Spec) bool {
 	}
 
 	var caps []string
-	// If v1.0.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
+	// If v1.1.0-rc1 <= OCI version < v1.0.0-rc5 parse s.Process.Capabilities as:
 	// github.com/opencontainers/runtime-spec/blob/v1.0.0-rc1/specs-go/config.go#L30-L54
 	rc1cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc1")
 	rc5cmp := semver.Compare("v"+*s.Version, "v1.0.0-rc5")
@@ -140,58 +167,73 @@ func isPrivileged(s *Spec) bool {
 		if err != nil {
 			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		for _, c := range caps {
-			if c == capSysAdmin {
-				return true
-			}
+		// Otherwise, parse s.Process.Capabilities as:
+		// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
+	} else {
+		var lc LinuxCapabilities
+		err := json.Unmarshal(*s.Process.Capabilities, &lc)
+		if err != nil {
+			log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
 		}
-		return false
+		// We only make sure that the bounding capabibility set has
+		// CAP_SYS_ADMIN. This allows us to make sure that the container was
+		// actually started as '--privileged', but also allow non-root users to
+		// access the privileged NVIDIA capabilities.
+		caps = lc.Bounding
 	}
 
-	// Otherwise, parse s.Process.Capabilities as:
-	// github.com/opencontainers/runtime-spec/blob/v1.0.0/specs-go/config.go#L30-L54
-	process := specs.Process{
-		Env: s.Process.Env,
+	for _, c := range caps {
+		if c == capSysAdmin {
+			return true
+		}
 	}
 
-	err := json.Unmarshal(*s.Process.Capabilities, &process.Capabilities)
-	if err != nil {
-		log.Panicln("could not decode Process.Capabilities in OCI spec:", err)
-	}
-
-	fullSpec := specs.Spec{
-		Version: *s.Version,
-		Process: &process,
-	}
-
-	return image.IsPrivileged(&fullSpec)
+	return false
 }
 
-func getDevicesFromEnvvar(image image.CUDA, swarmResourceEnvvars []string) *string {
-	// We check if the image has at least one of the Swarm resource envvars defined and use this
-	// if specified.
-	var hasSwarmEnvvar bool
-	for _, envvar := range swarmResourceEnvvars {
-		if _, exists := image[envvar]; exists {
-			hasSwarmEnvvar = true
+func isLegacyCUDAImage(env map[string]string) bool {
+	legacyCudaVersion := env[envCUDAVersion]
+	cudaRequire := env[envNVRequireCUDA]
+	return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0
+}
+
+func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
+	// Build a list of envvars to consider.
+	envVars := []string{envNVVisibleDevices}
+	if envSwarmGPU != nil {
+		// The Swarm envvar has higher precedence.
+		envVars = append([]string{*envSwarmGPU}, envVars...)
+	}
+
+	// Grab a reference to devices from the first envvar
+	// in the list that actually exists in the environment.
+	var devices *string
+	for _, envVar := range envVars {
+		if devs, ok := env[envVar]; ok {
+			devices = &devs
 			break
 		}
 	}
 
-	var devices []string
-	if hasSwarmEnvvar {
-		devices = image.DevicesFromEnvvars(swarmResourceEnvvars...).List()
-	} else {
-		devices = image.DevicesFromEnvvars(envNVVisibleDevices).List()
+	// Environment variable unset with legacy image: default to "all".
+	if devices == nil && legacyImage {
+		all := "all"
+		return &all
 	}
 
-	if len(devices) == 0 {
+	// Environment variable unset or empty or "void": return nil
+	if devices == nil || len(*devices) == 0 || *devices == "void" {
 		return nil
 	}
 
-	devicesString := strings.Join(devices, ",")
+	// Environment variable set to "none": reset to "".
+	if *devices == "none" {
+		empty := ""
+		return &empty
+	}
 
-	return &devicesString
+	// Any other value.
+	return devices
 }
 
 func getDevicesFromMounts(mounts []Mount) *string {
@@ -231,7 +273,7 @@ func getDevicesFromMounts(mounts []Mount) *string {
 	return &ret
 }
 
-func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *string {
+func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string {
 	// If enabled, try and get the device list from volume mounts first
 	if hookConfig.AcceptDeviceListAsVolumeMounts {
 		devices := getDevicesFromMounts(mounts)
@@ -241,7 +283,7 @@ func getDevices(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privil
 	}
 
 	// Fallback to reading from the environment variable if privileges are correct
-	devices := getDevicesFromEnvvar(image, hookConfig.getSwarmResourceEnvvars())
+	devices := getDevicesFromEnvvar(env, legacyImage)
 	if devices == nil {
 		return nil
 	}
@@ -293,11 +335,27 @@ func getDriverCapabilities(env map[string]string, supportedDriverCapabilities Dr
 	return capabilities
 }
 
-func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *nvidiaConfig {
-	legacyImage := image.IsLegacy()
+func getRequirements(env map[string]string, legacyImage bool) []string {
+	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
+	var requirements []string
+	for name, value := range env {
+		if strings.HasPrefix(name, envNVRequirePrefix) {
+			requirements = append(requirements, value)
+		}
+	}
+	if legacyImage {
+		vmaj, vmin, _ := parseCudaVersion(env[envCUDAVersion])
+		cudaRequire := fmt.Sprintf("cuda>=%d.%d", vmaj, vmin)
+		requirements = append(requirements, cudaRequire)
+	}
+	return requirements
+}
+
+func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool) *nvidiaConfig {
+	legacyImage := isLegacyCUDAImage(env)
 
 	var devices string
-	if d := getDevices(hookConfig, image, mounts, privileged); d != nil {
+	if d := getDevices(hookConfig, env, mounts, privileged, legacyImage); d != nil {
 		devices = *d
 	} else {
 		// 'nil' devices means this is not a GPU container.
@@ -305,7 +363,7 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 	}
 
 	var migConfigDevices string
-	if d := getMigConfigDevices(image); d != nil {
+	if d := getMigConfigDevices(env); d != nil {
 		migConfigDevices = *d
 	}
 	if !privileged && migConfigDevices != "" {
@@ -313,21 +371,19 @@ func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, p
 	}
 
 	var migMonitorDevices string
-	if d := getMigMonitorDevices(image); d != nil {
+	if d := getMigMonitorDevices(env); d != nil {
 		migMonitorDevices = *d
 	}
 	if !privileged && migMonitorDevices != "" {
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	driverCapabilities := getDriverCapabilities(image, hookConfig.SupportedDriverCapabilities, legacyImage).String()
+	driverCapabilities := getDriverCapabilities(env, hookConfig.SupportedDriverCapabilities, legacyImage).String()
 
-	requirements, err := image.GetRequirements()
-	if err != nil {
-		log.Panicln("failed to get requirements", err)
-	}
+	requirements := getRequirements(env, legacyImage)
 
-	disableRequire := image.HasDisableRequire()
+	// Don't fail on invalid values.
+	disableRequire, _ := strconv.ParseBool(env[envNVDisableRequire])
 
 	return &nvidiaConfig{
 		Devices:            devices,
@@ -353,16 +409,13 @@ func getContainerConfig(hook HookConfig) (config containerConfig) {
 
 	s := loadSpec(path.Join(b, "config.json"))
 
-	image, err := image.NewCUDAImageFromEnv(s.Process.Env)
-	if err != nil {
-		log.Panicln(err)
-	}
-
+	env := getEnvMap(s.Process.Env)
 	privileged := isPrivileged(s)
+	envSwarmGPU = hook.SwarmResource
 	return containerConfig{
 		Pid:    h.Pid,
 		Rootfs: s.Root.Path,
-		Env:    image,
-		Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged),
+		Env:    env,
+		Nvidia: getNvidiaConfig(&hook, env, s.Mounts, privileged),
 	}
 }

cmd/nvidia-container-toolkit/container_config_test.go

@@ -1,11 +1,9 @@
 package main
 
 import (
-	"fmt"
 	"path/filepath"
 	"testing"
 
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
 	"github.com/stretchr/testify/require"
 )
 
@@ -70,7 +68,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			description: "Legacy image, devices 'void', no capabilities, no requirements",
 			env: map[string]string{
 				envCUDAVersion:      "9.0",
-				envNVVisibleDevices: "void",
+				envNVVisibleDevices: "",
 			},
 			privileged:     false,
 			expectedConfig: nil,
@@ -227,7 +225,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			description: "Modern image, devices 'void', no capabilities, no requirements",
 			env: map[string]string{
 				envNVRequireCUDA:    "cuda>=9.0",
-				envNVVisibleDevices: "void",
+				envNVVisibleDevices: "",
 			},
 			privileged:     false,
 			expectedConfig: nil,
@@ -450,44 +448,6 @@ func TestGetNvidiaConfig(t *testing.T) {
 				DriverCapabilities: defaultDriverCapabilities.String(),
 			},
 		},
-		{
-			description: "Hook config set, swarmResource overrides device selection",
-			env: map[string]string{
-				envNVVisibleDevices:     "all",
-				"DOCKER_SWARM_RESOURCE": "GPU1,GPU2",
-			},
-			privileged: true,
-			hookConfig: &HookConfig{
-				SwarmResource: func() *string {
-					s := "DOCKER_SWARM_RESOURCE"
-					return &s
-				}(),
-				SupportedDriverCapabilities: "video,display,utility,compute",
-			},
-			expectedConfig: &nvidiaConfig{
-				Devices:            "GPU1,GPU2",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-			},
-		},
-		{
-			description: "Hook config set, comma separated swarmResource is split and overrides device selection",
-			env: map[string]string{
-				envNVVisibleDevices:     "all",
-				"DOCKER_SWARM_RESOURCE": "GPU1,GPU2",
-			},
-			privileged: true,
-			hookConfig: &HookConfig{
-				SwarmResource: func() *string {
-					s := "NOT_DOCKER_SWARM_RESOURCE,DOCKER_SWARM_RESOURCE"
-					return &s
-				}(),
-				SupportedDriverCapabilities: "video,display,utility,compute",
-			},
-			expectedConfig: &nvidiaConfig{
-				Devices:            "GPU1,GPU2",
-				DriverCapabilities: defaultDriverCapabilities.String(),
-			},
-		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
@@ -711,7 +671,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
 				hookConfig := getDefaultHookConfig()
 				hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
 				hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
-				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged)
+				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
 			}
 
 			// For all other tests, just grab the devices and check the results
@@ -728,13 +688,13 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
 	gpuID := "GPU-12345"
 	anotherGPUID := "GPU-67890"
-	thirdGPUID := "MIG-12345"
 
 	var tests = []struct {
-		description          string
-		swarmResourceEnvvars []string
-		env                  map[string]string
-		expectedDevices      *string
+		description     string
+		envSwarmGPU     *string
+		env             map[string]string
+		legacyImage     bool
+		expectedDevices *string
 	}{
 		{
 			description: "empty env returns nil for non-legacy image",
@@ -769,15 +729,13 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
 			env: map[string]string{
 				envNVVisibleDevices: gpuID,
-				envCUDAVersion:      "legacy",
 			},
+			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description: "empty env returns all for legacy image",
-			env: map[string]string{
-				envCUDAVersion: "legacy",
-			},
+			description:     "empty env returns all for legacy image",
+			legacyImage:     true,
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
@@ -823,116 +781,86 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
-				envCUDAVersion:        "legacy",
 			},
+			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
 			description: "empty env returns all for legacy image",
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
-				envCUDAVersion:        "legacy",
 			},
+			legacyImage:     true,
 			expectedDevices: &all,
 		},
 		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
 		// enabled
 		{
-			description:          "empty env returns nil for non-legacy image",
-			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			description: "empty env returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
 		},
 		{
-			description:          "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			description: "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
 			env: map[string]string{
 				envDockerResourceGPUs: "",
 			},
 		},
 		{
-			description:          "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
-			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			description: "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
 			env: map[string]string{
 				envDockerResourceGPUs: "void",
 			},
 		},
 		{
-			description:          "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
-			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			description: "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
 			env: map[string]string{
 				envDockerResourceGPUs: "none",
 			},
 			expectedDevices: &empty,
 		},
 		{
-			description:          "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
-			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			description: "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
 			},
 			expectedDevices: &gpuID,
 		},
 		{
-			description:          "DOCKER_RESOURCE_GPUS set returns value for legacy image",
-			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			description: "DOCKER_RESOURCE_GPUS set returns value for legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
 			env: map[string]string{
 				envDockerResourceGPUs: gpuID,
-				envCUDAVersion:        "legacy",
 			},
+			legacyImage:     true,
 			expectedDevices: &gpuID,
 		},
 		{
-			description:          "DOCKER_RESOURCE_GPUS is selected if present",
-			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			description: "DOCKER_RESOURCE_GPUS is selected if present",
+			envSwarmGPU: &envDockerResourceGPUs,
 			env: map[string]string{
 				envDockerResourceGPUs: anotherGPUID,
 			},
 			expectedDevices: &anotherGPUID,
 		},
 		{
-			description:          "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
-			swarmResourceEnvvars: []string{envDockerResourceGPUs},
+			description: "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
+			envSwarmGPU: &envDockerResourceGPUs,
 			env: map[string]string{
 				envNVVisibleDevices:   gpuID,
 				envDockerResourceGPUs: anotherGPUID,
 			},
 			expectedDevices: &anotherGPUID,
 		},
-		{
-			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL overrides NVIDIA_VISIBLE_DEVICES if present",
-			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS_ADDITIONAL"},
-			env: map[string]string{
-				envNVVisibleDevices:               gpuID,
-				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
-			},
-			expectedDevices: &anotherGPUID,
-		},
-		{
-			description:          "All available swarm resource envvars are selected and override NVIDIA_VISIBLE_DEVICES if present",
-			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
-			env: map[string]string{
-				envNVVisibleDevices:               gpuID,
-				"DOCKER_RESOURCE_GPUS":            thirdGPUID,
-				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
-			},
-			expectedDevices: func() *string {
-				result := fmt.Sprintf("%s,%s", thirdGPUID, anotherGPUID)
-				return &result
-			}(),
-		},
-		{
-			description:          "DOCKER_RESOURCE_GPUS_ADDITIONAL or DOCKER_RESOURCE_GPUS override NVIDIA_VISIBLE_DEVICES if present",
-			swarmResourceEnvvars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
-			env: map[string]string{
-				envNVVisibleDevices:               gpuID,
-				"DOCKER_RESOURCE_GPUS_ADDITIONAL": anotherGPUID,
-			},
-			expectedDevices: &anotherGPUID,
-		},
 	}
 
 	for i, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			devices := getDevicesFromEnvvar(image.CUDA(tc.env), tc.swarmResourceEnvvars)
+			envSwarmGPU = tc.envSwarmGPU
+			devices := getDevicesFromEnvvar(tc.env, tc.legacyImage)
 			if tc.expectedDevices == nil {
 				require.Nil(t, devices, "%d: %v", i, tc)
 				return

cmd/nvidia-container-toolkit/hook_config.go

@@ -5,10 +5,8 @@ import (
 	"os"
 	"path"
 	"reflect"
-	"strings"
 
 	"github.com/BurntSushi/toml"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
 )
 
 const (
@@ -35,7 +33,7 @@ type CLIConfig struct {
 	Ldconfig    *string  `toml:"ldconfig"`
 }
 
-// HookConfig : options for the nvidia-container-runtime-hook.
+// HookConfig : options for the nvidia-container-toolkit.
 type HookConfig struct {
 	DisableRequire                 bool               `toml:"disable-require"`
 	SwarmResource                  *string            `toml:"swarm-resource"`
@@ -43,12 +41,10 @@ type HookConfig struct {
 	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
 	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
 
-	NvidiaContainerCLI         CLIConfig                `toml:"nvidia-container-cli"`
-	NVIDIAContainerRuntime     config.RuntimeConfig     `toml:"nvidia-container-runtime"`
-	NVIDIAContainerRuntimeHook config.RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
+	NvidiaContainerCLI CLIConfig `toml:"nvidia-container-cli"`
 }
 
-func getDefaultHookConfig() HookConfig {
+func getDefaultHookConfig() (config HookConfig) {
 	return HookConfig{
 		DisableRequire:                 false,
 		SwarmResource:                  nil,
@@ -67,8 +63,6 @@ func getDefaultHookConfig() HookConfig {
 			User:        nil,
 			Ldconfig:    nil,
 		},
-		NVIDIAContainerRuntime:     *config.GetDefaultRuntimeConfig(),
-		NVIDIAContainerRuntimeHook: *config.GetDefaultRuntimeHookConfig(),
 	}
 }
 
@@ -119,22 +113,3 @@ func (c HookConfig) getConfigOption(fieldName string) string {
 	}
 	return v
 }
-
-// getSwarmResourceEnvvars returns the swarm resource envvars for the config.
-func (c *HookConfig) getSwarmResourceEnvvars() []string {
-	if c.SwarmResource == nil {
-		return nil
-	}
-
-	candidates := strings.Split(*c.SwarmResource, ",")
-
-	var envvars []string
-	for _, c := range candidates {
-		trimmed := strings.TrimSpace(c)
-		if len(trimmed) > 0 {
-			envvars = append(envvars, trimmed)
-		}
-	}
-
-	return envvars
-}

cmd/nvidia-container-toolkit/hook_config_test.go

@@ -103,59 +103,3 @@ func TestGetHookConfig(t *testing.T) {
 		})
 	}
 }
-
-func TestGetSwarmResourceEnvvars(t *testing.T) {
-	testCases := []struct {
-		value    string
-		expected []string
-	}{
-		{
-			value:    "nil",
-			expected: nil,
-		},
-		{
-			value:    "",
-			expected: nil,
-		},
-		{
-			value:    " ",
-			expected: nil,
-		},
-		{
-			value:    "single",
-			expected: []string{"single"},
-		},
-		{
-			value:    "single ",
-			expected: []string{"single"},
-		},
-		{
-			value:    "one,two",
-			expected: []string{"one", "two"},
-		},
-		{
-			value:    "one ,two",
-			expected: []string{"one", "two"},
-		},
-		{
-			value:    "one, two",
-			expected: []string{"one", "two"},
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("%d", i), func(t *testing.T) {
-			c := &HookConfig{
-				SwarmResource: func() *string {
-					if tc.value == "nil" {
-						return nil
-					}
-					return &tc.value
-				}(),
-			}
-
-			envvars := c.getSwarmResourceEnvvars()
-			require.EqualValues(t, tc.expected, envvars)
-		})
-	}
-}

cmd/nvidia-container-toolkit/hook_test.go

@@ -7,6 +7,51 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
+func TestParseCudaVersionValid(t *testing.T) {
+	var tests = []struct {
+		version  string
+		expected [3]uint32
+	}{
+		{"0", [3]uint32{0, 0, 0}},
+		{"8", [3]uint32{8, 0, 0}},
+		{"7.5", [3]uint32{7, 5, 0}},
+		{"9.0.116", [3]uint32{9, 0, 116}},
+		{"4294967295.4294967295.4294967295", [3]uint32{4294967295, 4294967295, 4294967295}},
+	}
+	for i, c := range tests {
+		vmaj, vmin, vpatch := parseCudaVersion(c.version)
+
+		version := [3]uint32{vmaj, vmin, vpatch}
+
+		require.Equal(t, c.expected, version, "%d: %v", i, c)
+	}
+}
+
+func TestParseCudaVersionInvalid(t *testing.T) {
+	var tests = []string{
+		"foo",
+		"foo.5.10",
+		"9.0.116.50",
+		"9.0.116foo",
+		"7.foo",
+		"9.0.bar",
+		"9.4294967296",
+		"9.0.116.",
+		"9..0",
+		"9.",
+		".5.10",
+		"-9",
+		"+9",
+		"-9.1.116",
+		"-9.-1.-116",
+	}
+	for _, c := range tests {
+		require.Panics(t, func() {
+			parseCudaVersion(c)
+		}, "parseCudaVersion(%v)", c)
+	}
+}
+
 func TestIsPrivileged(t *testing.T) {
 	var tests = []struct {
 		spec     string

cmd/nvidia-container-toolkit/main.go

@@ -6,21 +6,20 @@ import (
 	"log"
 	"os"
 	"os/exec"
+	"path"
 	"path/filepath"
 	"runtime"
 	"runtime/debug"
 	"strconv"
 	"strings"
 	"syscall"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
 var (
-	debugflag   = flag.Bool("debug", false, "enable debug output")
-	versionflag = flag.Bool("version", false, "enable version output")
-	configflag  = flag.String("config", "", "configuration file")
+	debugflag  = flag.Bool("debug", false, "enable debug output")
+	configflag = flag.String("config", "", "configuration file")
+
+	defaultPATH = []string{"/usr/local/sbin", "/usr/local/bin", "/usr/sbin", "/usr/bin", "/sbin", "/bin"}
 )
 
 func exit() {
@@ -36,16 +35,28 @@ func exit() {
 	os.Exit(0)
 }
 
+func getPATH(config CLIConfig) string {
+	dirs := filepath.SplitList(os.Getenv("PATH"))
+	// directories from the hook environment have higher precedence
+	dirs = append(dirs, defaultPATH...)
+
+	if config.Root != nil {
+		rootDirs := []string{}
+		for _, dir := range dirs {
+			rootDirs = append(rootDirs, path.Join(*config.Root, dir))
+		}
+		// directories with the root prefix have higher precedence
+		dirs = append(rootDirs, dirs...)
+	}
+	return strings.Join(dirs, ":")
+}
+
 func getCLIPath(config CLIConfig) string {
 	if config.Path != nil {
 		return *config.Path
 	}
 
-	var root string
-	if config.Root != nil {
-		root = *config.Root
-	}
-	if err := os.Setenv("PATH", lookup.GetPath(root)); err != nil {
+	if err := os.Setenv("PATH", getPATH(config)); err != nil {
 		log.Panicln("couldn't set PATH variable:", err)
 	}
 
@@ -74,10 +85,6 @@ func doPrestart() {
 	hook := getHookConfig()
 	cli := hook.NvidiaContainerCLI
 
-	if !hook.NVIDIAContainerRuntimeHook.SkipModeDetection && info.ResolveAutoMode(&logInterceptor{}, hook.NVIDIAContainerRuntime.Mode) != "legacy" {
-		log.Panicln("invoking the NVIDIA Container Runtime Hook directly (e.g. specifying the docker --gpus flag) is not supported. Please use the NVIDIA Container Runtime (e.g. specify the --runtime=nvidia flag) instead.")
-	}
-
 	container := getContainerConfig(hook)
 	nvidia := container.Nvidia
 	if nvidia == nil {
@@ -160,11 +167,6 @@ func main() {
 	flag.Usage = usage
 	flag.Parse()
 
-	if *versionflag {
-		fmt.Printf("%v version %v\n", "NVIDIA Container Runtime Hook", info.GetVersionString())
-		return
-	}
-
 	args := flag.Args()
 	if len(args) == 0 {
 		flag.Usage()
@@ -184,12 +186,3 @@ func main() {
 		os.Exit(2)
 	}
 }
-
-// logInterceptor implements the info.Logger interface to allow for logging from this function.
-type logInterceptor struct{}
-
-func (l *logInterceptor) Infof(format string, args ...interface{}) {
-	log.Printf(format, args...)
-}
-
-func (l *logInterceptor) Debugf(format string, args ...interface{}) {}

cmd/nvidia-ctk/README.md

@@ -1,48 +0,0 @@
-# NVIDIA Container Toolkit CLI
-
-The NVIDIA Container Toolkit CLI `nvidia-ctk` provides a number of utilities that are useful for working with the NVIDIA Container Toolkit.
-
-## Functionality
-
-### Configure runtimes
-
-The `runtime` command of the `nvidia-ctk` CLI provides a set of utilities to related to the configuration
-and management of supported container engines.
-
-For example, running the following command:
-```bash
-nvidia-ctk runtime configure --set-as-default
-```
-will ensure that the NVIDIA Container Runtime is added as the default runtime to the default container
-engine.
-
-### Generate CDI specifications
-
-The [Container Device Interface (CDI)](https://github.com/container-orchestrated-devices/container-device-interface) provides
-a vendor-agnostic mechanism to make arbitrary devices accessible in containerized environments. To allow NVIDIA devices to be
-used in these environments, the NVIDIA Container Toolkit CLI includes functionality to generate a CDI specification for the
-available NVIDIA GPUs in a system.
-
-In order to generate the CDI specification for the available devices, run the following command:\
-```bash
-nvidia-ctk cdi generate
-```
-
-The default is to print the specification to STDOUT and a filename can be specified using the `--output` flag.
-
-The specification will contain a device entries as follows (where applicable):
-* An `nvidia.com/gpu=gpu{INDEX}` device for each non-MIG-enabled full GPU in the system
-* An `nvidia.com/gpu=mig{GPU_INDEX}:{MIG_INDEX}` device for each MIG-device in the system
-* A special device called `nvidia.com/gpu=all` which represents all available devices.
-
-For example, to generate the CDI specification in the default location where CDI-enabled tools such as `podman`, `containerd`, `cri-o`, or the NVIDIA Container Runtime can be configured to load it, the following command can be run:
-
-```bash
-sudo nvidia-ctk cdi generate --output=/etc/cdi/nvidia.yaml
-```
-(Note that `sudo` is used to ensure the correct permissions to write to the `/etc/cdi` folder)
-
-With the specification generated, a GPU can be requested by specifying the fully-qualified CDI device name. With `podman` as an exmaple:
-```bash
-podman run --rm -ti --device=nvidia.com/gpu=gpu0 ubuntu nvidia-smi -L
-```

cmd/nvidia-ctk/cdi/cdi.go

@@ -1,52 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package cdi
-
-import (
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/generate"
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-// NewCommand constructs an info command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build
-func (m command) build() *cli.Command {
-	// Create the 'hook' command
-	hook := cli.Command{
-		Name:  "cdi",
-		Usage: "Provide tools for interacting with Container Device Interface specifications",
-	}
-
-	hook.Subcommands = []*cli.Command{
-		generate.NewCommand(m.logger),
-		transform.NewCommand(m.logger),
-	}
-
-	return &hook
-}

cmd/nvidia-ctk/cdi/generate/generate.go

@@ -1,283 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package generate
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
-	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi"
-	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
-	specs "github.com/container-orchestrated-devices/container-device-interface/specs-go"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-const (
-	allDeviceName = "all"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type config struct {
-	output             string
-	format             string
-	deviceNameStrategy string
-	driverRoot         string
-	nvidiaCTKPath      string
-	mode               string
-	vendor             string
-	class              string
-}
-
-// NewCommand constructs a generate-cdi command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build creates the CLI command
-func (m command) build() *cli.Command {
-	cfg := config{}
-
-	// Create the 'generate-cdi' command
-	c := cli.Command{
-		Name:  "generate",
-		Usage: "Generate CDI specifications for use with CDI-enabled runtimes",
-		Before: func(c *cli.Context) error {
-			return m.validateFlags(c, &cfg)
-		},
-		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "output",
-			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
-			Destination: &cfg.output,
-		},
-		&cli.StringFlag{
-			Name:        "format",
-			Usage:       "The output format for the generated spec [json | yaml]. This overrides the format defined by the output file extension (if specified).",
-			Value:       spec.FormatYAML,
-			Destination: &cfg.format,
-		},
-		&cli.StringFlag{
-			Name:        "mode",
-			Aliases:     []string{"discovery-mode"},
-			Usage:       "The mode to use when discovering the available entities. One of [auto | nvml | wsl]. If mode is set to 'auto' the mode will be determined based on the system configuration.",
-			Value:       nvcdi.ModeAuto,
-			Destination: &cfg.mode,
-		},
-		&cli.StringFlag{
-			Name:        "device-name-strategy",
-			Usage:       "Specify the strategy for generating device names. One of [index | uuid | type-index]",
-			Value:       nvcdi.DeviceNameStrategyIndex,
-			Destination: &cfg.deviceNameStrategy,
-		},
-		&cli.StringFlag{
-			Name:        "driver-root",
-			Usage:       "Specify the NVIDIA GPU driver root to use when discovering the entities that should be included in the CDI specification.",
-			Destination: &cfg.driverRoot,
-		},
-		&cli.StringFlag{
-			Name:        "nvidia-ctk-path",
-			Usage:       "Specify the path to use for the nvidia-ctk in the generated CDI specification. If this is left empty, the path will be searched.",
-			Destination: &cfg.nvidiaCTKPath,
-		},
-		&cli.StringFlag{
-			Name:        "vendor",
-			Aliases:     []string{"cdi-vendor"},
-			Usage:       "the vendor string to use for the generated CDI specification.",
-			Value:       "nvidia.com",
-			Destination: &cfg.vendor,
-		},
-		&cli.StringFlag{
-			Name:        "class",
-			Aliases:     []string{"cdi-class"},
-			Usage:       "the class string to use for the generated CDI specification.",
-			Value:       "gpu",
-			Destination: &cfg.class,
-		},
-	}
-
-	return &c
-}
-
-func (m command) validateFlags(c *cli.Context, cfg *config) error {
-
-	cfg.format = strings.ToLower(cfg.format)
-	switch cfg.format {
-	case spec.FormatJSON:
-	case spec.FormatYAML:
-	default:
-		return fmt.Errorf("invalid output format: %v", cfg.format)
-	}
-
-	cfg.mode = strings.ToLower(cfg.mode)
-	switch cfg.mode {
-	case nvcdi.ModeAuto:
-	case nvcdi.ModeNvml:
-	case nvcdi.ModeWsl:
-	case nvcdi.ModeManagement:
-	default:
-		return fmt.Errorf("invalid discovery mode: %v", cfg.mode)
-	}
-
-	_, err := nvcdi.NewDeviceNamer(cfg.deviceNameStrategy)
-	if err != nil {
-		return err
-	}
-
-	cfg.nvidiaCTKPath = discover.FindNvidiaCTK(m.logger, cfg.nvidiaCTKPath)
-
-	if outputFileFormat := formatFromFilename(cfg.output); outputFileFormat != "" {
-		m.logger.Debugf("Inferred output format as %q from output file name", outputFileFormat)
-		if !c.IsSet("format") {
-			cfg.format = outputFileFormat
-		} else if outputFileFormat != cfg.format {
-			m.logger.Warningf("Requested output format %q does not match format implied by output file name: %q", cfg.format, outputFileFormat)
-		}
-	}
-
-	if err := cdi.ValidateVendorName(cfg.vendor); err != nil {
-		return fmt.Errorf("invalid CDI vendor name: %v", err)
-	}
-	if err := cdi.ValidateClassName(cfg.class); err != nil {
-		return fmt.Errorf("invalid CDI class name: %v", err)
-	}
-	return nil
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	spec, err := m.generateSpec(cfg)
-	if err != nil {
-		return fmt.Errorf("failed to generate CDI spec: %v", err)
-	}
-	m.logger.Infof("Generated CDI spec with version %v", spec.Raw().Version)
-
-	if cfg.output == "" {
-		_, err := spec.WriteTo(os.Stdout)
-		if err != nil {
-			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
-		}
-		return nil
-	}
-
-	return spec.Save(cfg.output)
-}
-
-func formatFromFilename(filename string) string {
-	ext := filepath.Ext(filename)
-	switch strings.ToLower(ext) {
-	case ".json":
-		return spec.FormatJSON
-	case ".yaml", ".yml":
-		return spec.FormatYAML
-	}
-
-	return ""
-}
-
-func (m command) generateSpec(cfg *config) (spec.Interface, error) {
-	deviceNamer, err := nvcdi.NewDeviceNamer(cfg.deviceNameStrategy)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create device namer: %v", err)
-	}
-
-	cdilib, err := nvcdi.New(
-		nvcdi.WithLogger(m.logger),
-		nvcdi.WithDriverRoot(cfg.driverRoot),
-		nvcdi.WithNVIDIACTKPath(cfg.nvidiaCTKPath),
-		nvcdi.WithDeviceNamer(deviceNamer),
-		nvcdi.WithMode(string(cfg.mode)),
-	)
-	if err != nil {
-		return nil, fmt.Errorf("failed to create CDI library: %v", err)
-	}
-
-	deviceSpecs, err := cdilib.GetAllDeviceSpecs()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create device CDI specs: %v", err)
-	}
-	var hasAll bool
-	for _, deviceSpec := range deviceSpecs {
-		if deviceSpec.Name == allDeviceName {
-			hasAll = true
-			break
-		}
-	}
-	if !hasAll {
-		allDevice, err := MergeDeviceSpecs(deviceSpecs, allDeviceName)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create CDI specification for %q device: %v", allDeviceName, err)
-		}
-		deviceSpecs = append(deviceSpecs, allDevice)
-	}
-
-	commonEdits, err := cdilib.GetCommonEdits()
-	if err != nil {
-		return nil, fmt.Errorf("failed to create edits common for entities: %v", err)
-	}
-
-	return spec.New(
-		spec.WithVendor(cfg.vendor),
-		spec.WithClass(cfg.class),
-		spec.WithDeviceSpecs(deviceSpecs),
-		spec.WithEdits(*commonEdits.ContainerEdits),
-		spec.WithFormat(cfg.format),
-	)
-}
-
-// MergeDeviceSpecs creates a device with the specified name which combines the edits from the previous devices.
-// If a device of the specified name already exists, an error is returned.
-func MergeDeviceSpecs(deviceSpecs []specs.Device, mergedDeviceName string) (specs.Device, error) {
-	if err := cdi.ValidateDeviceName(mergedDeviceName); err != nil {
-		return specs.Device{}, fmt.Errorf("invalid device name %q: %v", mergedDeviceName, err)
-	}
-	for _, d := range deviceSpecs {
-		if d.Name == mergedDeviceName {
-			return specs.Device{}, fmt.Errorf("device %q already exists", mergedDeviceName)
-		}
-	}
-
-	mergedEdits := edits.NewContainerEdits()
-
-	for _, d := range deviceSpecs {
-		edit := cdi.ContainerEdits{
-			ContainerEdits: &d.ContainerEdits,
-		}
-		mergedEdits.Append(&edit)
-	}
-
-	merged := specs.Device{
-		Name:           mergedDeviceName,
-		ContainerEdits: *mergedEdits.ContainerEdits,
-	}
-	return merged, nil
-}

cmd/nvidia-ctk/cdi/generate/generate_test.go

@@ -1,117 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package generate
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
-	"github.com/stretchr/testify/require"
-)
-
-func TestMergeDeviceSpecs(t *testing.T) {
-	testCases := []struct {
-		description      string
-		deviceSpecs      []specs.Device
-		mergedDeviceName string
-		expectedError    error
-		expected         specs.Device
-	}{
-		{
-			description:      "no devices",
-			mergedDeviceName: "all",
-			expected: specs.Device{
-				Name: "all",
-			},
-		},
-		{
-			description:      "one device",
-			mergedDeviceName: "all",
-			deviceSpecs: []specs.Device{
-				{
-					Name: "gpu0",
-					ContainerEdits: specs.ContainerEdits{
-						Env: []string{"GPU=0"},
-					},
-				},
-			},
-			expected: specs.Device{
-				Name: "all",
-				ContainerEdits: specs.ContainerEdits{
-					Env: []string{"GPU=0"},
-				},
-			},
-		},
-		{
-			description:      "two devices",
-			mergedDeviceName: "all",
-			deviceSpecs: []specs.Device{
-				{
-					Name: "gpu0",
-					ContainerEdits: specs.ContainerEdits{
-						Env: []string{"GPU=0"},
-					},
-				},
-				{
-					Name: "gpu1",
-					ContainerEdits: specs.ContainerEdits{
-						Env: []string{"GPU=1"},
-					},
-				},
-			},
-			expected: specs.Device{
-				Name: "all",
-				ContainerEdits: specs.ContainerEdits{
-					Env: []string{"GPU=0", "GPU=1"},
-				},
-			},
-		},
-		{
-			description:      "has merged device",
-			mergedDeviceName: "gpu0",
-			deviceSpecs: []specs.Device{
-				{
-					Name: "gpu0",
-					ContainerEdits: specs.ContainerEdits{
-						Env: []string{"GPU=0"},
-					},
-				},
-			},
-			expectedError: fmt.Errorf("device %q already exists", "gpu0"),
-		},
-		{
-			description:      "invalid merged device name",
-			mergedDeviceName: ".-not-valid",
-			expectedError:    fmt.Errorf("invalid device name %q", ".-not-valid"),
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			mergedDevice, err := MergeDeviceSpecs(tc.deviceSpecs, tc.mergedDeviceName)
-
-			if tc.expectedError != nil {
-				require.Error(t, err)
-				return
-			}
-
-			require.NoError(t, err)
-			require.EqualValues(t, tc.expected, mergedDevice)
-		})
-	}
-}

cmd/nvidia-ctk/cdi/transform/root/root.go

@@ -1,159 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package root
-
-import (
-	"fmt"
-	"io"
-	"os"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/spec"
-	"github.com/NVIDIA/nvidia-container-toolkit/pkg/nvcdi/transform"
-	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type loadSaver interface {
-	Load() (spec.Interface, error)
-	Save(spec.Interface) error
-}
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type transformOptions struct {
-	input  string
-	output string
-}
-
-type options struct {
-	transformOptions
-	from string
-	to   string
-}
-
-// NewCommand constructs a generate-cdi command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build creates the CLI command
-func (m command) build() *cli.Command {
-	opts := options{}
-
-	c := cli.Command{
-		Name:  "root",
-		Usage: "Apply a root transform to a CDI specification",
-		Before: func(c *cli.Context) error {
-			return m.validateFlags(c, &opts)
-		},
-		Action: func(c *cli.Context) error {
-			return m.run(c, &opts)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "input",
-			Usage:       "Specify the file to read the CDI specification from. If this is '-' the specification is read from STDIN",
-			Value:       "-",
-			Destination: &opts.input,
-		},
-		&cli.StringFlag{
-			Name:        "output",
-			Usage:       "Specify the file to output the generated CDI specification to. If this is '' the specification is output to STDOUT",
-			Destination: &opts.output,
-		},
-		&cli.StringFlag{
-			Name:        "from",
-			Usage:       "specify the root to be transformed",
-			Destination: &opts.from,
-		},
-		&cli.StringFlag{
-			Name:        "to",
-			Usage:       "specify the replacement root. If this is the same as the from root, the transform is a no-op.",
-			Value:       "",
-			Destination: &opts.to,
-		},
-	}
-
-	return &c
-}
-
-func (m command) validateFlags(c *cli.Context, opts *options) error {
-	return nil
-}
-
-func (m command) run(c *cli.Context, opts *options) error {
-	spec, err := opts.Load()
-	if err != nil {
-		return fmt.Errorf("failed to load CDI specification: %w", err)
-	}
-
-	err = transform.NewRootTransformer(
-		opts.from,
-		opts.to,
-	).Transform(spec.Raw())
-	if err != nil {
-		return fmt.Errorf("failed to transform CDI specification: %w", err)
-	}
-
-	return opts.Save(spec)
-}
-
-// Load lodas the input CDI specification
-func (o transformOptions) Load() (spec.Interface, error) {
-	contents, err := o.getContents()
-	if err != nil {
-		return nil, fmt.Errorf("failed to read spec contents: %v", err)
-	}
-
-	raw, err := cdi.ParseSpec(contents)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse CDI spec: %v", err)
-	}
-
-	return spec.New(
-		spec.WithRawSpec(raw),
-	)
-}
-
-func (o transformOptions) getContents() ([]byte, error) {
-	if o.input == "-" {
-		return io.ReadAll(os.Stdin)
-	}
-
-	return os.ReadFile(o.input)
-}
-
-// Save saves the CDI specification to the output file
-func (o transformOptions) Save(s spec.Interface) error {
-	if o.output == "" {
-		_, err := s.WriteTo(os.Stdout)
-		if err != nil {
-			return fmt.Errorf("failed to write CDI spec to STDOUT: %v", err)
-		}
-		return nil
-	}
-
-	return s.Save(o.output)
-}

cmd/nvidia-ctk/cdi/transform/transform.go

@@ -1,51 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package transform
-
-import (
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi/transform/root"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-// NewCommand constructs a command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build creates the CLI command
-func (m command) build() *cli.Command {
-	c := cli.Command{
-		Name:  "transform",
-		Usage: "Apply a transform to a CDI specification",
-	}
-
-	c.Flags = []cli.Flag{}
-
-	c.Subcommands = []*cli.Command{
-		root.NewCommand(m.logger),
-	}
-
-	return &c
-}

cmd/nvidia-ctk/hook/chmod/chmod.go

@@ -1,146 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package chmod
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"syscall"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type config struct {
-	paths         cli.StringSlice
-	mode          string
-	containerSpec string
-}
-
-// NewCommand constructs a chmod command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build the chmod command
-func (m command) build() *cli.Command {
-	cfg := config{}
-
-	// Create the 'chmod' command
-	c := cli.Command{
-		Name:  "chmod",
-		Usage: "Set the permissions of folders in the container by running chmod. The container root is prefixed to the specified paths.",
-		Before: func(c *cli.Context) error {
-			return validateFlags(c, &cfg)
-		},
-		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringSliceFlag{
-			Name:        "path",
-			Usage:       "Specifiy a path to apply the specified mode to",
-			Destination: &cfg.paths,
-		},
-		&cli.StringFlag{
-			Name:        "mode",
-			Usage:       "Specify the file mode",
-			Destination: &cfg.mode,
-		},
-		&cli.StringFlag{
-			Name:        "container-spec",
-			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
-			Destination: &cfg.containerSpec,
-		},
-	}
-
-	return &c
-}
-
-func validateFlags(c *cli.Context, cfg *config) error {
-	if strings.TrimSpace(cfg.mode) == "" {
-		return fmt.Errorf("a non-empty mode must be specified")
-	}
-
-	for _, p := range cfg.paths.Value() {
-		if strings.TrimSpace(p) == "" {
-			return fmt.Errorf("paths must not be empty")
-		}
-	}
-
-	return nil
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	s, err := oci.LoadContainerState(cfg.containerSpec)
-	if err != nil {
-		return fmt.Errorf("failed to load container state: %v", err)
-	}
-
-	containerRoot, err := s.GetContainerRoot()
-	if err != nil {
-		return fmt.Errorf("failed to determined container root: %v", err)
-	}
-	if containerRoot == "" {
-		return fmt.Errorf("empty container root detected")
-	}
-
-	paths := m.getPaths(containerRoot, cfg.paths.Value())
-	if len(paths) == 0 {
-		m.logger.Debugf("No paths specified; exiting")
-		return nil
-	}
-
-	locator := lookup.NewExecutableLocator(m.logger, "")
-	targets, err := locator.Locate("chmod")
-	if err != nil {
-		return fmt.Errorf("failed to locate chmod: %v", err)
-	}
-	chmodPath := targets[0]
-
-	args := append([]string{filepath.Base(chmodPath), cfg.mode}, paths...)
-
-	return syscall.Exec(chmodPath, args, nil)
-}
-
-// getPaths updates the specified paths relative to the root.
-func (m command) getPaths(root string, paths []string) []string {
-	var pathsInRoot []string
-	for _, f := range paths {
-		path := filepath.Join(root, f)
-		if _, err := os.Stat(path); err != nil {
-			m.logger.Debugf("Skipping path %q: %v", path, err)
-			continue
-		}
-		pathsInRoot = append(pathsInRoot, path)
-	}
-
-	return pathsInRoot
-}

cmd/nvidia-ctk/hook/create-symlinks/create-symlinks.go

@@ -1,229 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package symlinks
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type config struct {
-	hostRoot      string
-	filenames     cli.StringSlice
-	links         cli.StringSlice
-	containerSpec string
-}
-
-// NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build
-func (m command) build() *cli.Command {
-	cfg := config{}
-
-	// Create the '' command
-	c := cli.Command{
-		Name:  "create-symlinks",
-		Usage: "A hook to create symlinks in the container. This can be used to proces CSV mount specs",
-		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "host-root",
-			Usage:       "The root on the host filesystem to use to resolve symlinks",
-			Destination: &cfg.hostRoot,
-		},
-		&cli.StringSliceFlag{
-			Name:        "csv-filename",
-			Usage:       "Specify a (CSV) filename to process",
-			Destination: &cfg.filenames,
-		},
-		&cli.StringSliceFlag{
-			Name:        "link",
-			Usage:       "Specify a specific link to create. The link is specified as target::link",
-			Destination: &cfg.links,
-		},
-		&cli.StringFlag{
-			Name:        "container-spec",
-			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
-			Destination: &cfg.containerSpec,
-		},
-	}
-
-	return &c
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	s, err := oci.LoadContainerState(cfg.containerSpec)
-	if err != nil {
-		return fmt.Errorf("failed to load container state: %v", err)
-	}
-
-	containerRoot, err := s.GetContainerRoot()
-	if err != nil {
-		return fmt.Errorf("failed to determined container root: %v", err)
-	}
-
-	csvFiles := cfg.filenames.Value()
-
-	chainLocator := lookup.NewSymlinkChainLocator(m.logger, cfg.hostRoot)
-
-	var candidates []string
-	for _, file := range csvFiles {
-		mountSpecs, err := csv.NewCSVFileParser(m.logger, file).Parse()
-		if err != nil {
-			m.logger.Debugf("Skipping CSV file %v: %v", file, err)
-			continue
-		}
-
-		for _, ms := range mountSpecs {
-			if ms.Type != csv.MountSpecSym {
-				continue
-			}
-			targets, err := chainLocator.Locate(ms.Path)
-			if err != nil {
-				m.logger.Warnf("Failed to locate symlink %v", ms.Path)
-			}
-			candidates = append(candidates, targets...)
-		}
-	}
-
-	created := make(map[string]bool)
-	// candidates is a list of absolute paths to symlinks in a chain, or the final target of the chain.
-	for _, candidate := range candidates {
-		targets, err := m.Locate(candidate)
-		if err != nil {
-			m.logger.Debugf("Skipping invalid link: %v", err)
-			continue
-		} else if len(targets) != 1 {
-			m.logger.Debugf("Unexepected number of targets: %v", targets)
-			continue
-		} else if targets[0] == candidate {
-			m.logger.Debugf("%v is not a symlink", candidate)
-			continue
-		}
-
-		err = m.createLink(created, cfg.hostRoot, containerRoot, targets[0], candidate)
-		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", []string{targets[0], candidate}, err)
-		}
-	}
-
-	links := cfg.links.Value()
-	for _, l := range links {
-		parts := strings.Split(l, "::")
-		if len(parts) != 2 {
-			m.logger.Warnf("Invalid link specification %v", l)
-			continue
-		}
-
-		err := m.createLink(created, cfg.hostRoot, containerRoot, parts[0], parts[1])
-		if err != nil {
-			m.logger.Warnf("Failed to create link %v: %v", parts, err)
-		}
-	}
-
-	return nil
-
-}
-
-func (m command) createLink(created map[string]bool, hostRoot string, containerRoot string, target string, link string) error {
-	linkPath, err := changeRoot(hostRoot, containerRoot, link)
-	if err != nil {
-		m.logger.Warnf("Failed to resolve path for link %v relative to %v: %v", link, containerRoot, err)
-	}
-	if created[linkPath] {
-		m.logger.Debugf("Link %v already created", linkPath)
-		return nil
-	}
-
-	targetPath, err := changeRoot(hostRoot, "/", target)
-	if err != nil {
-		m.logger.Warnf("Failed to resolve path for target %v relative to %v: %v", target, "/", err)
-	}
-
-	m.logger.Infof("Symlinking %v to %v", linkPath, targetPath)
-	err = os.MkdirAll(filepath.Dir(linkPath), 0755)
-	if err != nil {
-		return fmt.Errorf("failed to create directory: %v", err)
-	}
-	err = os.Symlink(target, linkPath)
-	if err != nil {
-		return fmt.Errorf("failed to create symlink: %v", err)
-	}
-
-	return nil
-}
-
-func changeRoot(current string, new string, path string) (string, error) {
-	if !filepath.IsAbs(path) {
-		return path, nil
-	}
-
-	relative := path
-	if current != "" {
-		r, err := filepath.Rel(current, path)
-		if err != nil {
-			return "", err
-		}
-		relative = r
-	}
-
-	return filepath.Join(new, relative), nil
-}
-
-// Locate returns the link target of the specified filename or an empty slice if the
-// specified filename is not a symlink.
-func (m command) Locate(filename string) ([]string, error) {
-	info, err := os.Lstat(filename)
-	if err != nil {
-		return nil, fmt.Errorf("failed to get file info: %v", info)
-	}
-	if info.Mode()&os.ModeSymlink == 0 {
-		m.logger.Debugf("%v is not a symlink", filename)
-		return nil, nil
-	}
-
-	target, err := os.Readlink(filename)
-	if err != nil {
-		return nil, fmt.Errorf("error checking symlink: %v", err)
-	}
-
-	m.logger.Debugf("Resolved link: '%v' => '%v'", filename, target)
-
-	return []string{target}, nil
-}

cmd/nvidia-ctk/hook/hook.go

@@ -1,55 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package hook
-
-import (
-	chmod "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/chmod"
-
-	symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/create-symlinks"
-	ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook/update-ldcache"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type hookCommand struct {
-	logger *logrus.Logger
-}
-
-// NewCommand constructs a hook command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := hookCommand{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build
-func (m hookCommand) build() *cli.Command {
-	// Create the 'hook' command
-	hook := cli.Command{
-		Name:  "hook",
-		Usage: "A collection of hooks that may be injected into an OCI spec",
-	}
-
-	hook.Subcommands = []*cli.Command{
-		ldcache.NewCommand(m.logger),
-		symlinks.NewCommand(m.logger),
-		chmod.NewCommand(m.logger),
-	}
-
-	return &hook
-}

cmd/nvidia-ctk/hook/update-ldcache/update-ldcache.go

@@ -1,129 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package ldcache
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"syscall"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type config struct {
-	folders       cli.StringSlice
-	containerSpec string
-}
-
-// NewCommand constructs an update-ldcache command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build the update-ldcache command
-func (m command) build() *cli.Command {
-	cfg := config{}
-
-	// Create the 'update-ldcache' command
-	c := cli.Command{
-		Name:  "update-ldcache",
-		Usage: "Update ldcache in a container by running ldconfig",
-		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringSliceFlag{
-			Name:        "folder",
-			Usage:       "Specifiy a folder to add to /etc/ld.so.conf before updating the ld cache",
-			Destination: &cfg.folders,
-		},
-		&cli.StringFlag{
-			Name:        "container-spec",
-			Usage:       "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
-			Destination: &cfg.containerSpec,
-		},
-	}
-
-	return &c
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	s, err := oci.LoadContainerState(cfg.containerSpec)
-	if err != nil {
-		return fmt.Errorf("failed to load container state: %v", err)
-	}
-
-	containerRoot, err := s.GetContainerRoot()
-	if err != nil {
-		return fmt.Errorf("failed to determined container root: %v", err)
-	}
-
-	err = m.createConfig(containerRoot, cfg.folders.Value())
-	if err != nil {
-		return fmt.Errorf("failed to update ld.so.conf: %v", err)
-	}
-
-	args := []string{"/sbin/ldconfig"}
-	if containerRoot != "" {
-		args = append(args, "-r", containerRoot)
-	}
-
-	return syscall.Exec(args[0], args, nil)
-}
-
-// createConfig creates (or updates) /etc/ld.so.conf.d/nvcr-<RANDOM_STRING>.conf in the container
-// to include the required paths.
-func (m command) createConfig(root string, folders []string) error {
-	if len(folders) == 0 {
-		m.logger.Debugf("No folders to add to /etc/ld.so.conf")
-		return nil
-	}
-
-	configFile, err := os.CreateTemp(filepath.Join(root, "/etc/ld.so.conf.d"), "nvcr-*.conf")
-	if err != nil {
-		return fmt.Errorf("failed to create config file: %v", err)
-	}
-	defer configFile.Close()
-
-	m.logger.Debugf("Adding folders %v to %v", folders, configFile.Name())
-
-	configured := make(map[string]bool)
-	for _, folder := range folders {
-		if configured[folder] {
-			continue
-		}
-		_, err = configFile.WriteString(fmt.Sprintf("%s\n", folder))
-		if err != nil {
-			return fmt.Errorf("failed to update ld.so.conf.d: %v", err)
-		}
-		configured[folder] = true
-	}
-
-	return nil
-}

cmd/nvidia-ctk/info/info.go

@@ -1,47 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package info
-
-import (
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-// NewCommand constructs an info command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build
-func (m command) build() *cli.Command {
-	// Create the 'hook' command
-	hook := cli.Command{
-		Name:  "info",
-		Usage: "Provide information about the system",
-	}
-
-	hook.Subcommands = []*cli.Command{}
-
-	return &hook
-}

cmd/nvidia-ctk/main.go

@@ -1,90 +0,0 @@
-/**
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package main
-
-import (
-	"os"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/cdi"
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/hook"
-	infoCLI "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/info"
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime"
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
-
-	log "github.com/sirupsen/logrus"
-	cli "github.com/urfave/cli/v2"
-)
-
-var logger = log.New()
-
-// config defines the options that can be set for the CLI through config files,
-// environment variables, or command line flags
-type config struct {
-	// Debug indicates whether the CLI is started in "debug" mode
-	Debug bool
-}
-
-func main() {
-	// Create a config struct to hold the parsed environment variables or command line flags
-	config := config{}
-
-	// Create the top-level CLI
-	c := cli.NewApp()
-	c.Name = "NVIDIA Container Toolkit CLI"
-	c.UseShortOptionHandling = true
-	c.EnableBashCompletion = true
-	c.Usage = "Tools to configure the NVIDIA Container Toolkit"
-	c.Version = info.GetVersionString()
-
-	// Setup the flags for this command
-	c.Flags = []cli.Flag{
-		&cli.BoolFlag{
-			Name:        "debug",
-			Aliases:     []string{"d"},
-			Usage:       "Enable debug-level logging",
-			Destination: &config.Debug,
-			EnvVars:     []string{"NVIDIA_CTK_DEBUG"},
-		},
-	}
-
-	// Set log-level for all subcommands
-	c.Before = func(c *cli.Context) error {
-		logLevel := log.InfoLevel
-		if config.Debug {
-			logLevel = log.DebugLevel
-		}
-		logger.SetLevel(logLevel)
-		return nil
-	}
-
-	// Define the subcommands
-	c.Commands = []*cli.Command{
-		hook.NewCommand(logger),
-		runtime.NewCommand(logger),
-		infoCLI.NewCommand(logger),
-		cdi.NewCommand(logger),
-		system.NewCommand(logger),
-	}
-
-	// Run the CLI
-	err := c.Run(os.Args)
-	if err != nil {
-		log.Errorf("%v", err)
-		log.Exit(1)
-	}
-}

cmd/nvidia-ctk/runtime/configure/configure.go

@@ -1,213 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package configure
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/nvidia"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/crio"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine/docker"
-	"github.com/pelletier/go-toml"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-const (
-	defaultRuntime = "docker"
-
-	defaultDockerConfigFilePath = "/etc/docker/daemon.json"
-	defaultCrioConfigFilePath   = "/etc/crio/crio.conf"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-// NewCommand constructs an configure command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// config defines the options that can be set for the CLI through config files,
-// environment variables, or command line config
-type config struct {
-	dryRun         bool
-	runtime        string
-	configFilePath string
-	nvidiaOptions  nvidia.Options
-}
-
-func (m command) build() *cli.Command {
-	// Create a config struct to hold the parsed environment variables or command line flags
-	config := config{}
-
-	// Create the 'configure' command
-	configure := cli.Command{
-		Name:  "configure",
-		Usage: "Add a runtime to the specified container engine",
-		Action: func(c *cli.Context) error {
-			return m.configureWrapper(c, &config)
-		},
-	}
-
-	configure.Flags = []cli.Flag{
-		&cli.BoolFlag{
-			Name:        "dry-run",
-			Usage:       "update the runtime configuration as required but don't write changes to disk",
-			Destination: &config.dryRun,
-		},
-		&cli.StringFlag{
-			Name:        "runtime",
-			Usage:       "the target runtime engine. One of [crio, docker]",
-			Value:       defaultRuntime,
-			Destination: &config.runtime,
-		},
-		&cli.StringFlag{
-			Name:        "config",
-			Usage:       "path to the config file for the target runtime",
-			Destination: &config.configFilePath,
-		},
-		&cli.StringFlag{
-			Name:        "nvidia-runtime-name",
-			Usage:       "specify the name of the NVIDIA runtime that will be added",
-			Value:       nvidia.RuntimeName,
-			Destination: &config.nvidiaOptions.RuntimeName,
-		},
-		&cli.StringFlag{
-			Name:        "runtime-path",
-			Usage:       "specify the path to the NVIDIA runtime executable",
-			Value:       nvidia.RuntimeExecutable,
-			Destination: &config.nvidiaOptions.RuntimePath,
-		},
-		&cli.BoolFlag{
-			Name:        "set-as-default",
-			Usage:       "set the specified runtime as the default runtime",
-			Destination: &config.nvidiaOptions.SetAsDefault,
-		},
-	}
-
-	return &configure
-}
-
-func (m command) configureWrapper(c *cli.Context, config *config) error {
-	switch config.runtime {
-	case "crio":
-		return m.configureCrio(c, config)
-	case "docker":
-		return m.configureDocker(c, config)
-	}
-
-	return fmt.Errorf("unrecognized runtime '%v'", config.runtime)
-}
-
-// configureDocker updates the docker config to enable the NVIDIA Container Runtime
-func (m command) configureDocker(c *cli.Context, config *config) error {
-	configFilePath := config.configFilePath
-	if configFilePath == "" {
-		configFilePath = defaultDockerConfigFilePath
-	}
-
-	cfg, err := docker.New(
-		docker.WithPath(configFilePath),
-	)
-	if err != nil {
-		return fmt.Errorf("unable to load config: %v", err)
-	}
-
-	err = cfg.AddRuntime(
-		config.nvidiaOptions.RuntimeName,
-		config.nvidiaOptions.RuntimePath,
-		config.nvidiaOptions.SetAsDefault,
-	)
-	if err != nil {
-		return fmt.Errorf("unable to update config: %v", err)
-	}
-
-	if config.dryRun {
-		output, err := json.MarshalIndent(cfg, "", "    ")
-		if err != nil {
-			return fmt.Errorf("unable to convert to JSON: %v", err)
-		}
-		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
-		return nil
-	}
-	n, err := cfg.Save(configFilePath)
-	if err != nil {
-		return fmt.Errorf("unable to flush config: %v", err)
-	}
-
-	if n == 0 {
-		m.logger.Infof("Removed empty config from %v", configFilePath)
-	} else {
-		m.logger.Infof("Wrote updated config to %v", configFilePath)
-	}
-	m.logger.Infof("It is recommended that the docker daemon be restarted.")
-
-	return nil
-}
-
-// configureCrio updates the crio config to enable the NVIDIA Container Runtime
-func (m command) configureCrio(c *cli.Context, config *config) error {
-	configFilePath := config.configFilePath
-	if configFilePath == "" {
-		configFilePath = defaultCrioConfigFilePath
-	}
-
-	cfg, err := crio.New(
-		crio.WithPath(configFilePath),
-	)
-	if err != nil {
-		return fmt.Errorf("unable to load config: %v", err)
-	}
-
-	err = cfg.AddRuntime(
-		config.nvidiaOptions.RuntimeName,
-		config.nvidiaOptions.RuntimePath,
-		config.nvidiaOptions.SetAsDefault,
-	)
-	if err != nil {
-		return fmt.Errorf("unable to update config: %v", err)
-	}
-
-	if config.dryRun {
-		output, err := toml.Marshal(cfg)
-		if err != nil {
-			return fmt.Errorf("unable to convert to TOML: %v", err)
-		}
-		os.Stdout.WriteString(fmt.Sprintf("%s\n", output))
-		return nil
-	}
-	n, err := cfg.Save(configFilePath)
-	if err != nil {
-		return fmt.Errorf("unable to flush config: %v", err)
-	}
-
-	if n == 0 {
-		m.logger.Infof("Removed empty config from %v", configFilePath)
-	} else {
-		m.logger.Infof("Wrote updated config to %v", configFilePath)
-	}
-	m.logger.Infof("It is recommended that the cri-o daemon be restarted.")
-
-	return nil
-}

cmd/nvidia-ctk/runtime/nvidia/nvidia.go

@@ -1,75 +0,0 @@
-/*
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package nvidia
-
-const (
-	// RuntimeName is the default name to use in configs for the NVIDIA Container Runtime
-	RuntimeName = "nvidia"
-	// RuntimeExecutable is the default NVIDIA Container Runtime executable file name
-	RuntimeExecutable = "nvidia-container-runtime"
-)
-
-// Options specifies the options for the NVIDIA Container Runtime w.r.t a container engine such as docker.
-type Options struct {
-	SetAsDefault bool
-	RuntimeName  string
-	RuntimePath  string
-}
-
-// Runtime defines an NVIDIA runtime with a name and a executable
-type Runtime struct {
-	Name string
-	Path string
-}
-
-// DefaultRuntime returns the default runtime for the configured options.
-// If the configuration is invalid or the default runtimes should not be set
-// the empty string is returned.
-func (o Options) DefaultRuntime() string {
-	if !o.SetAsDefault {
-		return ""
-	}
-
-	return o.RuntimeName
-}
-
-// Runtime creates a runtime struct based on the options.
-func (o Options) Runtime() Runtime {
-	path := o.RuntimePath
-
-	if o.RuntimePath == "" {
-		path = RuntimeExecutable
-	}
-
-	r := Runtime{
-		Name: o.RuntimeName,
-		Path: path,
-	}
-
-	return r
-}
-
-// DockerRuntimesConfig generatest the expected docker config for the specified runtime
-func (r Runtime) DockerRuntimesConfig() map[string]interface{} {
-	runtimes := make(map[string]interface{})
-	runtimes[r.Name] = map[string]interface{}{
-		"path": r.Path,
-		"args": []string{},
-	}
-
-	return runtimes
-}

cmd/nvidia-ctk/runtime/runtime.go

@@ -1,49 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package runtime
-
-import (
-	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/runtime/configure"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type runtimeCommand struct {
-	logger *logrus.Logger
-}
-
-// NewCommand constructs a runtime command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := runtimeCommand{
-		logger: logger,
-	}
-	return c.build()
-}
-
-func (m runtimeCommand) build() *cli.Command {
-	// Create the 'runtime' command
-	runtime := cli.Command{
-		Name:  "runtime",
-		Usage: "A collection of runtime-related utilities for the NVIDIA Container Toolkit",
-	}
-
-	runtime.Subcommands = []*cli.Command{
-		configure.NewCommand(m.logger),
-	}
-
-	return &runtime
-}

cmd/nvidia-ctk/system/create-dev-char-symlinks/all.go

@@ -1,175 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package devchar
-
-import (
-	"fmt"
-	"path/filepath"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/nvcaps"
-	"github.com/sirupsen/logrus"
-	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci"
-)
-
-type allPossible struct {
-	logger       *logrus.Logger
-	driverRoot   string
-	deviceMajors devices.Devices
-	migCaps      nvcaps.MigCaps
-}
-
-// newAllPossible returns a new allPossible device node lister.
-// This lister lists all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
-func newAllPossible(logger *logrus.Logger, driverRoot string) (nodeLister, error) {
-	deviceMajors, err := devices.GetNVIDIADevices()
-	if err != nil {
-		return nil, fmt.Errorf("failed reading device majors: %v", err)
-	}
-	migCaps, err := nvcaps.NewMigCaps()
-	if err != nil {
-		return nil, fmt.Errorf("failed to read MIG caps: %v", err)
-	}
-	if migCaps == nil {
-		migCaps = make(nvcaps.MigCaps)
-	}
-
-	l := allPossible{
-		logger:       logger,
-		driverRoot:   driverRoot,
-		deviceMajors: deviceMajors,
-		migCaps:      migCaps,
-	}
-
-	return l, nil
-}
-
-// DeviceNodes returns a list of all possible device nodes for NVIDIA GPUs, control devices, and capability devices.
-func (m allPossible) DeviceNodes() ([]deviceNode, error) {
-	gpus, err := nvpci.NewFrom(
-		filepath.Join(m.driverRoot, nvpci.PCIDevicesRoot),
-	).GetGPUs()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get GPU information: %v", err)
-	}
-
-	count := len(gpus)
-	if count == 0 {
-		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
-		return nil, nil
-	}
-
-	deviceNodes, err := m.getControlDeviceNodes()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get control device nodes: %v", err)
-	}
-
-	for gpu := 0; gpu < count; gpu++ {
-		deviceNodes = append(deviceNodes, m.getGPUDeviceNodes(gpu)...)
-		deviceNodes = append(deviceNodes, m.getNVCapDeviceNodes(gpu)...)
-	}
-
-	return deviceNodes, nil
-}
-
-// getControlDeviceNodes generates a list of control devices
-func (m allPossible) getControlDeviceNodes() ([]deviceNode, error) {
-	var deviceNodes []deviceNode
-
-	// Define the control devices for standard GPUs.
-	controlDevices := []deviceNode{
-		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidia-modeset", devices.NVIDIAModesetMinor),
-		m.newDeviceNode(devices.NVIDIAGPU, "/dev/nvidiactl", devices.NVIDIACTLMinor),
-		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm", devices.NVIDIAUVMMinor),
-		m.newDeviceNode(devices.NVIDIAUVM, "/dev/nvidia-uvm-tools", devices.NVIDIAUVMToolsMinor),
-	}
-
-	deviceNodes = append(deviceNodes, controlDevices...)
-
-	for _, migControlDevice := range []nvcaps.MigCap{"config", "monitor"} {
-		migControlMinor, exist := m.migCaps[migControlDevice]
-		if !exist {
-			continue
-		}
-
-		d := m.newDeviceNode(
-			devices.NVIDIACaps,
-			migControlMinor.DevicePath(),
-			int(migControlMinor),
-		)
-
-		deviceNodes = append(deviceNodes, d)
-	}
-
-	return deviceNodes, nil
-}
-
-// getGPUDeviceNodes generates a list of device nodes for a given GPU.
-func (m allPossible) getGPUDeviceNodes(gpu int) []deviceNode {
-	d := m.newDeviceNode(
-		devices.NVIDIAGPU,
-		fmt.Sprintf("/dev/nvidia%d", gpu),
-		gpu,
-	)
-
-	return []deviceNode{d}
-}
-
-// getNVCapDeviceNodes generates a list of cap device nodes for a given GPU.
-func (m allPossible) getNVCapDeviceNodes(gpu int) []deviceNode {
-	var selectedCapMinors []nvcaps.MigMinor
-	for gi := 0; ; gi++ {
-		giCap := nvcaps.NewGPUInstanceCap(gpu, gi)
-		giMinor, exist := m.migCaps[giCap]
-		if !exist {
-			break
-		}
-		selectedCapMinors = append(selectedCapMinors, giMinor)
-		for ci := 0; ; ci++ {
-			ciCap := nvcaps.NewComputeInstanceCap(gpu, gi, ci)
-			ciMinor, exist := m.migCaps[ciCap]
-			if !exist {
-				break
-			}
-			selectedCapMinors = append(selectedCapMinors, ciMinor)
-		}
-	}
-
-	var deviceNodes []deviceNode
-	for _, capMinor := range selectedCapMinors {
-		d := m.newDeviceNode(
-			devices.NVIDIACaps,
-			capMinor.DevicePath(),
-			int(capMinor),
-		)
-		deviceNodes = append(deviceNodes, d)
-	}
-
-	return deviceNodes
-}
-
-// newDeviceNode creates a new device node with the specified path and major/minor numbers.
-// The path is adjusted for the specified driver root.
-func (m allPossible) newDeviceNode(deviceName devices.Name, path string, minor int) deviceNode {
-	major, _ := m.deviceMajors.Get(deviceName)
-
-	return deviceNode{
-		path:  filepath.Join(m.driverRoot, path),
-		major: uint32(major),
-		minor: uint32(minor),
-	}
-}

cmd/nvidia-ctk/system/create-dev-char-symlinks/create-dev-char-symlinks.go

@@ -1,332 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package devchar
-
-import (
-	"fmt"
-	"os"
-	"os/signal"
-	"path/filepath"
-	"strings"
-	"syscall"
-
-	"github.com/fsnotify/fsnotify"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-const (
-	defaultDevCharPath = "/dev/char"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type config struct {
-	devCharPath string
-	driverRoot  string
-	dryRun      bool
-	watch       bool
-	createAll   bool
-}
-
-// NewCommand constructs a command sub-command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build
-func (m command) build() *cli.Command {
-	cfg := config{}
-
-	// Create the 'create-dev-char-symlinks' command
-	c := cli.Command{
-		Name:  "create-dev-char-symlinks",
-		Usage: "A utility to create symlinks to possible /dev/nv* devices in /dev/char",
-		Before: func(c *cli.Context) error {
-			return m.validateFlags(c, &cfg)
-		},
-		Action: func(c *cli.Context) error {
-			return m.run(c, &cfg)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "dev-char-path",
-			Usage:       "The path at which the symlinks will be created. Symlinks will be created as `DEV_CHAR`/MAJOR:MINOR where MAJOR and MINOR are the major and minor numbers of a corresponding device node.",
-			Value:       defaultDevCharPath,
-			Destination: &cfg.devCharPath,
-			EnvVars:     []string{"DEV_CHAR_PATH"},
-		},
-		&cli.StringFlag{
-			Name:        "driver-root",
-			Usage:       "The path to the driver root. `DRIVER_ROOT`/dev is searched for NVIDIA device nodes.",
-			Value:       "/",
-			Destination: &cfg.driverRoot,
-			EnvVars:     []string{"DRIVER_ROOT"},
-		},
-		&cli.BoolFlag{
-			Name:        "watch",
-			Usage:       "If set, the command will watch for changes to the driver root and recreate the symlinks when changes are detected.",
-			Value:       false,
-			Destination: &cfg.watch,
-			EnvVars:     []string{"WATCH"},
-		},
-		&cli.BoolFlag{
-			Name:        "create-all",
-			Usage:       "Create all possible /dev/char symlinks instead of limiting these to existing device nodes.",
-			Destination: &cfg.createAll,
-			EnvVars:     []string{"CREATE_ALL"},
-		},
-		&cli.BoolFlag{
-			Name:        "dry-run",
-			Usage:       "If set, the command will not create any symlinks.",
-			Value:       false,
-			Destination: &cfg.dryRun,
-			EnvVars:     []string{"DRY_RUN"},
-		},
-	}
-
-	return &c
-}
-
-func (m command) validateFlags(r *cli.Context, cfg *config) error {
-	if cfg.createAll && cfg.watch {
-		return fmt.Errorf("create-all and watch are mutually exclusive")
-	}
-
-	return nil
-}
-
-func (m command) run(c *cli.Context, cfg *config) error {
-	var watcher *fsnotify.Watcher
-	var sigs chan os.Signal
-
-	if cfg.watch {
-		watcher, err := newFSWatcher(filepath.Join(cfg.driverRoot, "dev"))
-		if err != nil {
-			return fmt.Errorf("failed to create FS watcher: %v", err)
-		}
-		defer watcher.Close()
-
-		sigs = newOSWatcher(syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT)
-	}
-
-	l, err := NewSymlinkCreator(
-		WithLogger(m.logger),
-		WithDevCharPath(cfg.devCharPath),
-		WithDriverRoot(cfg.driverRoot),
-		WithDryRun(cfg.dryRun),
-		WithCreateAll(cfg.createAll),
-	)
-	if err != nil {
-		return fmt.Errorf("failed to create symlink creator: %v", err)
-	}
-
-create:
-	err = l.CreateLinks()
-	if err != nil {
-		return fmt.Errorf("failed to create links: %v", err)
-	}
-	if !cfg.watch {
-		return nil
-	}
-	for {
-		select {
-
-		case event := <-watcher.Events:
-			deviceNode := filepath.Base(event.Name)
-			if !strings.HasPrefix(deviceNode, "nvidia") {
-				continue
-			}
-			if event.Op&fsnotify.Create == fsnotify.Create {
-				m.logger.Infof("%s created, restarting.", event.Name)
-				goto create
-			}
-			if event.Op&fsnotify.Create == fsnotify.Remove {
-				m.logger.Infof("%s removed. Ignoring", event.Name)
-
-			}
-
-		// Watch for any other fs errors and log them.
-		case err := <-watcher.Errors:
-			m.logger.Errorf("inotify: %s", err)
-
-		// React to signals
-		case s := <-sigs:
-			switch s {
-			case syscall.SIGHUP:
-				m.logger.Infof("Received SIGHUP, recreating symlinks.")
-				goto create
-			default:
-				m.logger.Infof("Received signal %q, shutting down.", s)
-				return nil
-			}
-		}
-	}
-}
-
-type linkCreator struct {
-	logger      *logrus.Logger
-	lister      nodeLister
-	driverRoot  string
-	devCharPath string
-	dryRun      bool
-	createAll   bool
-}
-
-// Creator is an interface for creating symlinks to /dev/nv* devices in /dev/char.
-type Creator interface {
-	CreateLinks() error
-}
-
-// Option is a functional option for configuring the linkCreator.
-type Option func(*linkCreator)
-
-// NewSymlinkCreator creates a new linkCreator.
-func NewSymlinkCreator(opts ...Option) (Creator, error) {
-	c := linkCreator{}
-	for _, opt := range opts {
-		opt(&c)
-	}
-	if c.logger == nil {
-		c.logger = logrus.StandardLogger()
-	}
-	if c.driverRoot == "" {
-		c.driverRoot = "/"
-	}
-	if c.devCharPath == "" {
-		c.devCharPath = defaultDevCharPath
-	}
-
-	if c.createAll {
-		lister, err := newAllPossible(c.logger, c.driverRoot)
-		if err != nil {
-			return nil, fmt.Errorf("failed to create all possible device lister: %v", err)
-		}
-		c.lister = lister
-	} else {
-		c.lister = existing{c.logger, c.driverRoot}
-	}
-	return c, nil
-}
-
-// WithDriverRoot sets the driver root path.
-func WithDriverRoot(root string) Option {
-	return func(c *linkCreator) {
-		c.driverRoot = root
-	}
-}
-
-// WithDevCharPath sets the path at which the symlinks will be created.
-func WithDevCharPath(path string) Option {
-	return func(c *linkCreator) {
-		c.devCharPath = path
-	}
-}
-
-// WithDryRun sets the dry run flag.
-func WithDryRun(dryRun bool) Option {
-	return func(c *linkCreator) {
-		c.dryRun = dryRun
-	}
-}
-
-// WithLogger sets the logger.
-func WithLogger(logger *logrus.Logger) Option {
-	return func(c *linkCreator) {
-		c.logger = logger
-	}
-}
-
-// WithCreateAll sets the createAll flag for the linkCreator.
-func WithCreateAll(createAll bool) Option {
-	return func(lc *linkCreator) {
-		lc.createAll = createAll
-	}
-}
-
-// CreateLinks creates symlinks for all NVIDIA device nodes found in the driver root.
-func (m linkCreator) CreateLinks() error {
-	deviceNodes, err := m.lister.DeviceNodes()
-	if err != nil {
-		return fmt.Errorf("failed to get device nodes: %v", err)
-	}
-
-	if len(deviceNodes) != 0 && !m.dryRun {
-		err := os.MkdirAll(m.devCharPath, 0755)
-		if err != nil {
-			return fmt.Errorf("failed to create directory %s: %v", m.devCharPath, err)
-		}
-	}
-
-	for _, deviceNode := range deviceNodes {
-		target := deviceNode.path
-		linkPath := filepath.Join(m.devCharPath, deviceNode.devCharName())
-
-		m.logger.Infof("Creating link %s => %s", linkPath, target)
-		if m.dryRun {
-			continue
-		}
-
-		err = os.Symlink(target, linkPath)
-		if err != nil {
-			m.logger.Warnf("Could not create symlink: %v", err)
-		}
-	}
-
-	return nil
-}
-
-type deviceNode struct {
-	path  string
-	major uint32
-	minor uint32
-}
-
-func (d deviceNode) devCharName() string {
-	return fmt.Sprintf("%d:%d", d.major, d.minor)
-}
-
-func newFSWatcher(files ...string) (*fsnotify.Watcher, error) {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		return nil, err
-	}
-
-	for _, f := range files {
-		err = watcher.Add(f)
-		if err != nil {
-			watcher.Close()
-			return nil, err
-		}
-	}
-
-	return watcher, nil
-}
-
-func newOSWatcher(sigs ...os.Signal) chan os.Signal {
-	sigChan := make(chan os.Signal, 1)
-	signal.Notify(sigChan, sigs...)
-
-	return sigChan
-}

cmd/nvidia-ctk/system/create-dev-char-symlinks/existing.go

@@ -1,95 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package devchar
-
-import (
-	"path/filepath"
-	"strings"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/sys/unix"
-)
-
-type nodeLister interface {
-	DeviceNodes() ([]deviceNode, error)
-}
-
-type existing struct {
-	logger     *logrus.Logger
-	driverRoot string
-}
-
-// DeviceNodes returns a list of NVIDIA device nodes in the specified root.
-// The nvidia-nvswitch* and nvidia-nvlink devices are excluded.
-func (m existing) DeviceNodes() ([]deviceNode, error) {
-	locator := lookup.NewCharDeviceLocator(
-		lookup.WithLogger(m.logger),
-		lookup.WithRoot(m.driverRoot),
-		lookup.WithOptional(true),
-	)
-
-	devices, err := locator.Locate("/dev/nvidia*")
-	if err != nil {
-		m.logger.Warnf("Error while locating device: %v", err)
-	}
-
-	capDevices, err := locator.Locate("/dev/nvidia-caps/nvidia-*")
-	if err != nil {
-		m.logger.Warnf("Error while locating caps device: %v", err)
-	}
-
-	if len(devices) == 0 && len(capDevices) == 0 {
-		m.logger.Infof("No NVIDIA devices found in %s", m.driverRoot)
-		return nil, nil
-	}
-
-	var deviceNodes []deviceNode
-	for _, d := range append(devices, capDevices...) {
-		if m.nodeIsBlocked(d) {
-			continue
-		}
-
-		var stat unix.Stat_t
-		err := unix.Stat(d, &stat)
-		if err != nil {
-			m.logger.Warnf("Could not stat device: %v", err)
-			continue
-		}
-		deviceNode := deviceNode{
-			path:  d,
-			major: unix.Major(uint64(stat.Rdev)),
-			minor: unix.Minor(uint64(stat.Rdev)),
-		}
-
-		deviceNodes = append(deviceNodes, deviceNode)
-	}
-
-	return deviceNodes, nil
-}
-
-// nodeIsBlocked returns true if the specified device node should be ignored.
-func (m existing) nodeIsBlocked(path string) bool {
-	blockedPrefixes := []string{"nvidia-fs", "nvidia-nvswitch", "nvidia-nvlink"}
-	nodeName := filepath.Base(path)
-	for _, prefix := range blockedPrefixes {
-		if strings.HasPrefix(nodeName, prefix) {
-			return true
-		}
-	}
-	return false
-}

cmd/nvidia-ctk/system/create-device-nodes/create-device-nodes.go

@@ -1,107 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package createdevicenodes
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/system"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-type options struct {
-	driverRoot string
-
-	dryRun bool
-
-	control bool
-}
-
-// NewCommand constructs a command sub-command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-// build
-func (m command) build() *cli.Command {
-	opts := options{}
-
-	c := cli.Command{
-		Name:  "create-device-nodes",
-		Usage: "A utility to create NVIDIA device ndoes",
-		Before: func(c *cli.Context) error {
-			return m.validateFlags(c, &opts)
-		},
-		Action: func(c *cli.Context) error {
-			return m.run(c, &opts)
-		},
-	}
-
-	c.Flags = []cli.Flag{
-		&cli.StringFlag{
-			Name:        "driver-root",
-			Usage:       "the path to the driver root. Device nodes will be created at `DRIVER_ROOT`/dev",
-			Value:       "/",
-			Destination: &opts.driverRoot,
-			EnvVars:     []string{"DRIVER_ROOT"},
-		},
-		&cli.BoolFlag{
-			Name:        "control-devices",
-			Usage:       "create all control device nodes: nvidiactl, nvidia-modeset, nvidia-uvm, nvidia-uvm-tools",
-			Destination: &opts.control,
-		},
-		&cli.BoolFlag{
-			Name:        "dry-run",
-			Usage:       "if set, the command will not create any symlinks.",
-			Value:       false,
-			Destination: &opts.dryRun,
-			EnvVars:     []string{"DRY_RUN"},
-		},
-	}
-
-	return &c
-}
-
-func (m command) validateFlags(r *cli.Context, opts *options) error {
-	return nil
-}
-
-func (m command) run(c *cli.Context, opts *options) error {
-	s, err := system.New(
-		system.WithLogger(m.logger),
-		system.WithDryRun(opts.dryRun),
-	)
-	if err != nil {
-		return fmt.Errorf("failed to create library: %v", err)
-	}
-
-	if opts.control {
-		m.logger.Infof("Creating control device nodes at %s", opts.driverRoot)
-		if err := s.CreateNVIDIAControlDeviceNodesAt(opts.driverRoot); err != nil {
-			return fmt.Errorf("failed to create control device nodes: %v", err)
-		}
-	}
-	return nil
-}

cmd/nvidia-ctk/system/system.go

@@ -1,51 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package system
-
-import (
-	devchar "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-dev-char-symlinks"
-	devicenodes "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk/system/create-device-nodes"
-	"github.com/sirupsen/logrus"
-	"github.com/urfave/cli/v2"
-)
-
-type command struct {
-	logger *logrus.Logger
-}
-
-// NewCommand constructs a runtime command with the specified logger
-func NewCommand(logger *logrus.Logger) *cli.Command {
-	c := command{
-		logger: logger,
-	}
-	return c.build()
-}
-
-func (m command) build() *cli.Command {
-	// Create the 'system' command
-	system := cli.Command{
-		Name:  "system",
-		Usage: "A collection of system-related utilities for the NVIDIA Container Toolkit",
-	}
-
-	system.Subcommands = []*cli.Command{
-		devchar.NewCommand(m.logger),
-		devicenodes.NewCommand(m.logger),
-	}
-
-	return &system
-}

config/config.toml.amazonlinux

@@ -16,17 +16,3 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.centos

@@ -0,0 +1,18 @@
+disable-require = false
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#accept-nvidia-visible-devices-as-volume-mounts = false
+
+[nvidia-container-cli]
+#root = "/run/nvidia/driver"
+#path = "/usr/bin/nvidia-container-cli"
+environment = []
+#debug = "/var/log/nvidia-container-toolkit.log"
+#ldcache = "/etc/ld.so.cache"
+load-kmods = true
+#no-cgroups = false
+#user = "root:video"
+ldconfig = "@/sbin/ldconfig"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"

config/config.toml.debian

@@ -16,17 +16,3 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.opensuse-leap

@@ -16,17 +16,3 @@ ldconfig = "@/sbin/ldconfig"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.ubuntu

@@ -16,17 +16,3 @@ ldconfig = "@/sbin/ldconfig.real"
 
 [nvidia-container-runtime]
 #debug = "/var/log/nvidia-container-runtime.log"
-log-level = "info"
-
-# Specify the runtimes to consider. This list is processed in order and the PATH
-# searched for matching executables unless the entry is an absolute path.
-runtimes = [
-    "docker-runc",
-    "runc",
-]
-
-mode = "auto"
-
-    [nvidia-container-runtime.modes.csv]
-
-    mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d"

config/config.toml.ubuntu+jetpack

@@ -0,0 +1,19 @@
+disable-require = false
+supported-driver-capabilities = "compute,compat32,graphics,utility,video,display"
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#accept-nvidia-visible-devices-as-volume-mounts = false
+
+[nvidia-container-cli]
+#root = "/run/nvidia/driver"
+#path = "/usr/bin/nvidia-container-cli"
+environment = []
+#debug = "/var/log/nvidia-container-toolkit.log"
+#ldcache = "/etc/ld.so.cache"
+load-kmods = true
+#no-cgroups = false
+#user = "root:video"
+ldconfig = "@/sbin/ldconfig.real"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"

docker/Dockerfile.amazonlinux

@@ -1,29 +1,14 @@
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-# This is the dockerfile for building packages on yum-based RPM systems.
-
 ARG BASEIMAGE
 FROM ${BASEIMAGE}
 
 RUN yum install -y \
         ca-certificates \
         gcc \
-        wget \
         git \
         make \
-        rpm-build && \
+        rpm-build \
+        wget \
+        && \
     rm -rf /var/cache/yum/*
 
 ARG GOLANG_VERSION=0.0.0
@@ -43,12 +28,11 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
-ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-ENV PKG_NAME ${PKG_NAME}
-ENV PKG_VERS ${PKG_VERS}
-ENV PKG_REV ${PKG_REV}
+
+ENV VERSION $PKG_VERS
+ENV RELEASE $PKG_REV
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -58,8 +42,6 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-ARG GIT_COMMIT
-ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 ARG CONFIG_TOML_SUFFIX
@@ -67,6 +49,8 @@ ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
 COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
+# This might not be useful on Amazon Linux, but it's simpler to keep the RHEL
+# and Amazon Linux packages identical.
 COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
 
 # Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
@@ -75,16 +59,11 @@ COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
-ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
-ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
-
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
-             -D "release_date $(date +'%a %b %d %Y')" \
-             -D "git_commit ${GIT_COMMIT}" \
-             -D "version ${PKG_VERS}" \
-             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
-             -D "release ${PKG_REV}" \
+             -D "version $VERSION" \
+             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
+             -D "release $RELEASE" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.centos

@@ -0,0 +1,67 @@
+ARG BASEIMAGE
+FROM ${BASEIMAGE}
+
+RUN yum install -y \
+        ca-certificates \
+        gcc \
+        git \
+        make \
+        rpm-build \
+        wget \
+        && \
+    rm -rf /var/cache/yum/*
+
+ARG GOLANG_VERSION=0.0.0
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture"; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+ENV GOPATH /go
+ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
+
+# packaging
+ARG PKG_VERS
+ARG PKG_REV
+
+ENV VERSION $PKG_VERS
+ENV RELEASE $PKG_REV
+
+# output directory
+ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
+RUN mkdir -p $DIST_DIR /dist
+
+# nvidia-container-toolkit
+WORKDIR $GOPATH/src/nvidia-container-toolkit
+COPY . .
+
+RUN make PREFIX=${DIST_DIR} cmds
+
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
+
+# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
+COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
+
+# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
+COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
+
+WORKDIR $DIST_DIR/..
+COPY packaging/rpm .
+
+CMD arch=$(uname -m) && \
+    rpmbuild --clean --target=$arch -bb \
+             -D "_topdir $PWD" \
+             -D "version $VERSION" \
+             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
+             -D "release $RELEASE" \
+             SPECS/nvidia-container-toolkit.spec && \
+    mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.debian

@@ -32,7 +32,6 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
-ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
 
@@ -49,8 +48,6 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-ARG GIT_COMMIT
-ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 ARG CONFIG_TOML_SUFFIX
@@ -65,17 +62,12 @@ RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
-ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
-ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
-
-RUN dch --create --package="${PKG_NAME}" \
-        --newversion "${REVISION}" \
-            "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
-    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER1_VERSION}" && \
-    dch -r "" && \
+RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
+    dch --changelog debian/changelog -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
-    mv /tmp/*.deb /dist
+    mv /tmp/nvidia-container-toolkit_*.deb /dist

docker/Dockerfile.devel

@@ -14,8 +14,7 @@
 ARG GOLANG_VERSION=x.x.x
 FROM golang:${GOLANG_VERSION}
 
-RUN go install golang.org/x/lint/golint@6edffad5e6160f5949cdefc81710b2706fbcd4f6
-RUN go install github.com/matryer/moq@latest
-RUN go install github.com/gordonklaus/ineffassign@d2c82e48359b033cde9cf1307f6d5550b8d61321
-RUN go install github.com/client9/misspell/cmd/misspell@latest
-RUN go install github.com/google/go-licenses@latest
+RUN go get -u golang.org/x/lint/golint
+RUN go get -u github.com/matryer/moq
+RUN go get -u github.com/gordonklaus/ineffassign
+RUN go get -u github.com/client9/misspell/cmd/misspell

docker/Dockerfile.opensuse-leap

@@ -25,12 +25,11 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
-ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
-ENV PKG_NAME ${PKG_NAME}
-ENV PKG_VERS ${PKG_VERS}
-ENV PKG_REV ${PKG_REV}
+
+ENV VERSION $PKG_VERS
+ENV RELEASE $PKG_REV
 
 # output directory
 ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES
@@ -40,8 +39,6 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-ARG GIT_COMMIT
-ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
@@ -57,16 +54,11 @@ COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
 
-ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
-ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
-
 CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
-             -D "release_date $(date +'%a %b %d %Y')" \
-             -D "git_commit ${GIT_COMMIT}" \
-             -D "version ${PKG_VERS}" \
-             -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \
-             -D "release ${PKG_REV}" \
+             -D "version $VERSION" \
+             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
+             -D "release $RELEASE" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.ubuntu

@@ -30,7 +30,6 @@ ENV GOPATH /go
 ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH
 
 # packaging
-ARG PKG_NAME
 ARG PKG_VERS
 ARG PKG_REV
 
@@ -47,8 +46,6 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-ARG GIT_COMMIT
-ENV GIT_COMMIT ${GIT_COMMIT}
 RUN make PREFIX=${DIST_DIR} cmds
 
 ARG CONFIG_TOML_SUFFIX
@@ -58,17 +55,12 @@ COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
-ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION
-ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}
-
-RUN dch --create --package="${PKG_NAME}" \
-        --newversion "${REVISION}" \
-            "See https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/blob/${GIT_COMMIT}/CHANGELOG.md for the changelog" && \
-    dch --append "Bump libnvidia-container dependency to ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" && \
-    dch -r "" && \
+RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
+    dch --changelog debian/changelog -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_TOOLS_VERSION -eVERSION="${REVISION}" \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
     --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/*.deb /dist

docker/docker.mk

@@ -85,40 +85,34 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 --%: docker-build-%
 	@
 
-LIBNVIDIA_CONTAINER_VERSION ?= $(LIB_VERSION)
-LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG)
-
-LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)$(if $(LIBNVIDIA_CONTAINER_TAG),~$(LIBNVIDIA_CONTAINER_TAG))-1
-
 # private ubuntu target
 --ubuntu%: OS := ubuntu
+--ubuntu%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+--ubuntu%: PKG_REV := 1
 
 # private debian target
 --debian%: OS := debian
+--debian%: LIB_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+--debian%: PKG_REV := 1
 
 # private centos target
 --centos%: OS := centos
---centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---centos%: CONFIG_TOML_SUFFIX := rpm-yum
---centos8%: BASEIMAGE = quay.io/centos/centos:stream8
+--centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
---amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---amazonlinux%: CONFIG_TOML_SUFFIX := rpm-yum
+--amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
 --opensuse-leap%: BASEIMAGE = opensuse/leap:$(VERSION)
+--opensuse-leap%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
+--rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
---rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum
---rhel%: CONFIG_TOML_SUFFIX := rpm-yum
---rhel8%: BASEIMAGE = quay.io/centos/centos:stream8
-
 
 # We allow the CONFIG_TOML_SUFFIX to be overridden.
 CONFIG_TOML_SUFFIX ?= $(OS)
@@ -128,20 +122,15 @@ docker-build-%:
 	docker pull --platform=linux/$(ARCH) $(BASEIMAGE)
 	DOCKER_BUILDKIT=1 \
 	$(DOCKER) build \
-	    --platform=linux/$(ARCH) \
 	    --progress=plain \
 	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-	    --build-arg PKG_NAME="$(LIB_NAME)" \
-	    --build-arg PKG_VERS="$(PACKAGE_VERSION)" \
-	    --build-arg PKG_REV="$(PACKAGE_REVISION)" \
-	    --build-arg LIBNVIDIA_CONTAINER_TOOLS_VERSION="$(LIBNVIDIA_CONTAINER_TOOLS_VERSION)" \
+	    --build-arg PKG_VERS="$(LIB_VERSION)" \
+	    --build-arg PKG_REV="$(PKG_REV)" \
 	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
-	    --build-arg GIT_COMMIT="$(GIT_COMMIT)" \
 	    --tag $(BUILDIMAGE) \
 	    --file $(DOCKERFILE) .
 	$(DOCKER) run \
-	    --platform=linux/$(ARCH) \
 	    -e DISTRIB \
 	    -e SECTION \
 	    -v $(ARTIFACTS_DIR):/dist \

examples/discover/main.go

@@ -14,35 +14,43 @@
 # limitations under the License.
 */
 
-package lookup
+package main
 
 import (
-	"fmt"
+	"encoding/json"
 	"os"
 
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 	log "github.com/sirupsen/logrus"
 )
 
-// NewDirectoryLocator creates a Locator that can be used to find directories at the specified root. A logger
-// is also specified.
-func NewDirectoryLocator(logger *log.Logger, root string) Locator {
-	return NewFileLocator(
-		WithLogger(logger),
-		WithRoot(root),
-		WithFilter(assertDirectory),
-	)
-}
+func main() {
+	log.Infof("Starting device discovery with NVML")
 
-// assertDirectory checks wither the specified path is a directory.
-func assertDirectory(filename string) error {
-	info, err := os.Stat(filename)
+	d, err := discover.NewNVMLServer("")
 	if err != nil {
-		return fmt.Errorf("error getting info for %v: %v", filename, err)
+		log.Errorf("Error creating NVML Server: %v", err)
+		return
 	}
 
-	if !info.IsDir() {
-		return fmt.Errorf("specified path '%v' is not a directory", filename)
+	devices, err := d.Devices()
+	if err != nil {
+		log.Errorf("Error discovering devices: %v", err)
+		return
 	}
 
-	return nil
+	mounts, err := d.Mounts()
+	if err != nil {
+		log.Errorf("Error discovering mounts: %v", err)
+		return
+	}
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+
+	log.Infof("Discovered devices:")
+	enc.Encode(devices)
+
+	log.Infof("Discovered libraries:")
+	enc.Encode(mounts)
 }

examples/filter/main.go

@@ -0,0 +1,65 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"os"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/filter"
+	log "github.com/sirupsen/logrus"
+)
+
+func main() {
+	d, err := discover.NewNVMLServer("")
+	if err != nil {
+		log.Errorf("Error discovering devices: %v", err)
+	}
+
+	selected := filter.NewSelectDevicesFrom(d, "all", nil)
+
+	devices, err := selected.Devices()
+	if err != nil {
+		log.Errorf("Error discovering devices: %v", err)
+		return
+	}
+
+	mounts, err := selected.Mounts()
+	if err != nil {
+		log.Errorf("Error discovering mounts: %v", err)
+		return
+	}
+
+	hooks, err := selected.Hooks()
+	if err != nil {
+		log.Errorf("Error discovering hooks: %v", err)
+		return
+	}
+
+	enc := json.NewEncoder(os.Stdout)
+	enc.SetIndent("", "  ")
+
+	log.Infof("Discovered devices:")
+	enc.Encode(devices)
+
+	log.Infof("Discovered libraries:")
+	enc.Encode(mounts)
+
+	log.Infof("Discovered hook:")
+	enc.Encode(hooks)
+}

examples/print-ldcache/main.go

@@ -0,0 +1,26 @@
+package main
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+	log "github.com/sirupsen/logrus"
+)
+
+var logger = log.StandardLogger()
+
+func main() {
+	logger.SetLevel(log.DebugLevel)
+	logger.Infof("Starting device discovery with NVML")
+
+	cache, err := ldcache.NewLDCacheWithLogger(logger, "/run/nvidia/driver")
+	if err != nil {
+		logger.Errorf("Error loading ldcache: %v", err)
+		return
+	}
+	defer cache.Close()
+
+	libs32, libs64 := cache.Lookup("lib")
+
+	logger.Infof("32-bit: %v", libs32)
+	logger.Infof("64-bit: %v", libs64)
+
+}

go.mod

@@ -1,36 +1,17 @@
 module github.com/NVIDIA/nvidia-container-toolkit
 
-go 1.18
+go 1.14
 
 require (
-	github.com/BurntSushi/toml v1.0.0
-	github.com/NVIDIA/go-nvml v0.12.0-0
-	github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a
-	github.com/fsnotify/fsnotify v1.5.4
-	github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb
-	github.com/pelletier/go-toml v1.9.4
-	github.com/sirupsen/logrus v1.9.0
+	github.com/BurntSushi/toml v0.3.1
+	github.com/NVIDIA/go-nvml v0.11.1-0
+	github.com/containers/podman/v2 v2.2.1
+	github.com/opencontainers/runtime-spec v1.0.3-0.20211101234015-a3c33d663ebc
+	github.com/pelletier/go-toml v1.9.3
+	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
+	github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d
 	github.com/urfave/cli/v2 v2.3.0
-	gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438
-	golang.org/x/mod v0.5.0
-	golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6
-	sigs.k8s.io/yaml v1.3.0
-)
-
-require (
-	github.com/cpuguy83/go-md2man/v2 v2.0.1 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
-	github.com/hashicorp/errwrap v1.1.0 // indirect
-	github.com/kr/text v0.2.0 // indirect
-	github.com/opencontainers/runc v1.1.4 // indirect
-	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
-	github.com/opencontainers/selinux v1.10.1 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/russross/blackfriday/v2 v2.1.0 // indirect
-	github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 // indirect
-	github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect
-	gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	golang.org/x/mod v0.3.0
+	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
 )

go.sum

@@ -1,127 +1,751 @@
+bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
+github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v1.0.0 h1:dtDWrepsVPfW9H/4y7dDgFc2MBUSeJhlaDtK13CxFlU=
-github.com/BurntSushi/toml v1.0.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
-github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82 h1:x751Xx1tdxkiA/sdkv2J769n21UbYKzVOpe9S/h1M3k=
-github.com/NVIDIA/go-nvml v0.11.6-0.0.20220823120812-7e2082095e82/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
-github.com/NVIDIA/go-nvml v0.12.0-0 h1:eHYNHbzAsMgWYshf6dEmTY66/GCXnORJFnzm3TNH4mc=
-github.com/NVIDIA/go-nvml v0.12.0-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
-github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM=
-github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ=
-github.com/checkpoint-restore/go-criu/v5 v5.3.0/go.mod h1:E/eQpaFtUKGOOSEBZgmKAcn+zUUwWxqcaKZlF54wK8E=
-github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA=
-github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a h1:sP3PcgyIkRlHqfF3Jfpe/7G8kf/qpzG4C8r94y9hLbE=
-github.com/container-orchestrated-devices/container-device-interface v0.5.4-0.20230111111500-5b3b5d81179a/go.mod h1:xMRa4fJgXzSDFUCURSimOUgoSc+odohvO3uXT9xjqH0=
-github.com/containerd/console v1.0.3/go.mod h1:7LqA/THxQ86k76b8c/EMSiaJ3h1eZkMkXar0TQ1gf3U=
-github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/NVIDIA/go-nvml v0.11.1-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
+github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/buger/goterm v0.0.0-20181115115552-c206103e1f37/go.mod h1:u9UyCz2eTrSGy6fbupqJ54eY5c4IC8gREQ1053dK12U=
+github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho=
+github.com/checkpoint-restore/go-criu/v4 v4.0.2/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.0.0-20200507155900-a9f01edf17e3/go.mod h1:XT+cAw5wfvsodedcijoh1l9cf7v1x9FlFB/3VmF/O8s=
+github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
+github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
+github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
+github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
+github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY=
+github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
+github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
+github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
+github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
+github.com/containernetworking/cni v0.7.2-0.20190904153231-83439463f784/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0=
+github.com/containers/buildah v1.18.0/go.mod h1:qHLk7RUL7cHfA7ve1MKkZ6cyKUxHD0YxiLJcKY+mJe8=
+github.com/containers/common v0.26.3/go.mod h1:hJWZIlrl5MsE2ELNRa+MPp6I1kPbXHauuj0Ym4BsLG4=
+github.com/containers/common v0.29.0/go.mod h1:yT4GTUHsKRmpaDb+mecXRnIMre7W3ZgwXqaYMywXlaA=
+github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
+github.com/containers/image/v5 v5.7.0/go.mod h1:8aOy+YaItukxghRORkvhq5ibWttHErzDLy6egrKfKos=
+github.com/containers/image/v5 v5.8.0/go.mod h1:jKxdRtyIDumVa56hdsZvV+gwx4zB50hRou6pIuCWLkg=
+github.com/containers/image/v5 v5.8.1/go.mod h1:blOEFd/iFdeyh891ByhCVUc+xAcaI3gBegXECwz9UbQ=
+github.com/containers/image/v5 v5.9.0/go.mod h1:blOEFd/iFdeyh891ByhCVUc+xAcaI3gBegXECwz9UbQ=
+github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
+github.com/containers/ocicrypt v1.0.3/go.mod h1:CUBa+8MRNL/VkpxYIpaMtgn1WgXGyvPQj8jcy0EVG6g=
+github.com/containers/podman/v2 v2.2.1 h1:ONDOHuzYnPF+ZJ+sV9hjtssAG93gTbyvlKN0LhbFIRY=
+github.com/containers/podman/v2 v2.2.1/go.mod h1:4CuPT3c5jB1XxIjFRiAkqrvXrW+g5NR5wQb58u4KJE0=
+github.com/containers/psgo v1.5.1/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzPUWfawVU=
+github.com/containers/storage v1.23.6/go.mod h1:haFs0HRowKwyzvWEx9EgI3WsL8XCSnBDb5f8P5CAxJY=
+github.com/containers/storage v1.23.7/go.mod h1:cUT2zHjtx+WlVri30obWmM2gpqpi8jfPsmIzP1TVpEI=
+github.com/containers/storage v1.24.0/go.mod h1:A4d3BzuZK9b3oLVEsiSRhZLPIx3z7utgiPyXLK/YMhY=
+github.com/containers/storage v1.24.1/go.mod h1:0xJL06Dmd+ZYXIUdnBUPN0JnhHGgwMkLvnnAonJfWJU=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.1 h1:r/myEWzV9lfsM1tFLgDyu0atFtJ1fXn261LKYj/3DxU=
-github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
-github.com/cyphar/filepath-securejoin v0.2.3/go.mod h1:aPGpWjXOXUn2NCNjFvBE6aRxGGx79pTxQpKOJNYHHl4=
+github.com/cri-o/ocicni v0.2.1-0.20201102180012-75c612fda1a2/go.mod h1:vingr1ztOAzP2WyTgGbpMov9dFhbjNxdLtDv0+PhAvY=
+github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
+github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
+github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
+github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
+github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v17.12.0-ce-rc1.0.20200505174321-1655290016ac+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v17.12.0-ce-rc1.0.20201020191947-73dc6a680cdd+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
-github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k=
-github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI=
-github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU=
-github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsouza/go-dockerclient v1.6.6/go.mod h1:3/oRIWoe7uT6bwtAayj/EmJmepBjeL4pYvt7ZxC7Rnk=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
+github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
+github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
 github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
-github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
-github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
-github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
-github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/insomniacslk/dhcp v0.0.0-20200806210722-3f14f7f8bd9c/go.mod h1:CfMdguCK66I5DAUJgGKyNz8aB6vO5dZzkm9Xep6WGvw=
+github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
+github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
+github.com/jamescun/tuntap v0.0.0-20190712092105-cb1fb277045c/go.mod h1:zzwpsgcYhzzIP5WyF8g9ivCv38cY9uAV9Gu0m3lThhE=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.11.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
-github.com/mndrix/tap-go v0.0.0-20171203230836-629fa407e90b/go.mod h1:pzzDgJWZ34fGzaAZGFW22KVZDfyrYW+QABMrWnJBnSs=
-github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU=
-github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
-github.com/opencontainers/runc v1.1.4 h1:nRCz/8sKg6K6jgYAFLDlXzPeITBZJyX28DBVhWD+5dg=
-github.com/opencontainers/runc v1.1.4/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg=
-github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb h1:1xSVPOd7/UA+39/hXEGnBJ13p6JFB0E1EvQFlrRDOXI=
-github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
-github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 h1:DmNGcqH3WDbV5k8OJ+esPWbqUOX5rMLR2PMvziDMJi0=
-github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626/go.mod h1:BRHJJd0E+cx42OybVYSgUvZmU0B8P9gZuRXlZUP7TKI=
-github.com/opencontainers/selinux v1.9.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w=
-github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI=
-github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM=
-github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
+github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11/go.mod h1:Ah2dBMoxZEqk118as2T4u4fjfXarE0pPnMJaArZQZsI=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/sys/mount v0.1.0/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74=
+github.com/moby/sys/mount v0.1.1/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74=
+github.com/moby/sys/mountinfo v0.1.0/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
+github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
+github.com/moby/sys/mountinfo v0.3.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/term v0.0.0-20200429084858-129dac9f73f6/go.mod h1:or9wGItza1sRcM4Wd3dIv8DsFHYQuFsMHEdxUIlUxms=
+github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2/go.mod h1:TjQg8pa4iejrUrjiz0MCtMV38jdMNW4doKSiBrEvCQQ=
+github.com/moby/vpnkit v0.4.0/go.mod h1:KyjUrL9cb6ZSNNAUwZfqRjhwwgJ3BJN+kXh0t43WTUQ=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0=
+github.com/mtrmac/gpgme v0.1.2/go.mod h1:GYYHnGSuS7HK3zVS2n3y73y0okK/BeKzwnn5jgiVFNI=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v0.0.0-20190425234816-dae70e8efea4/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v1.0.0-rc91/go.mod h1:3Sm6Dt7OT8z88EbdQqqcRN2oCT54jbi72tT/HqgflT8=
+github.com/opencontainers/runc v1.0.0-rc91.0.20200708210054-ce54a9d4d79b/go.mod h1:ZuXhqlr4EiRYgDrBDNfSbE4+n9JX4+V107NwAmF7sZA=
+github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20200520003142-237cc4f519e2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20200817204227-f9c09b4ea1df/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20211101234015-a3c33d663ebc h1:Q4P71bRTYvUK/qLX897YrMBA1oznb1noH0rLB5ji9SE=
+github.com/opencontainers/runtime-spec v1.0.3-0.20211101234015-a3c33d663ebc/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
+github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
+github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
+github.com/openshift/imagebuilder v1.1.8/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc=
+github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.0.0-20190227000051-27936f6d90f9/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
+github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rootless-containers/rootlesskit v0.11.1/go.mod h1:pCUqFJBGOIonbjQBaxSVnk3w3KnK2drqjllgpgvNnO8=
+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk=
-github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
+github.com/seccomp/libseccomp-golang v0.9.2-0.20200616122406-847368b35ebf/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v0.0.0-20190403091019-9b3cdde74fbe/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
 github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
-github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
-github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
-github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI=
-github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
-github.com/urfave/cli v1.19.1/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d h1:hq9X/cf03C5rCx9yWhY7eMHiNxmhTMJAc5DQBq9BfnI=
+github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d/go.mod h1:oFPCwcQpP90RVZxlBdgPN+iu2tPkboPUa4xaVEI6pO4=
+github.com/tsaikd/govalidator v0.0.0-20161031084447-986f2244fc69/go.mod h1:yJymgtZhuWi1Ih5t37Ej381BGZFZvlb9YMTwBxB/QjU=
+github.com/u-root/u-root v6.0.0+incompatible/go.mod h1:RYkpo8pTHrNjW08opNd/U6p/RJE7K0D8fXO0d47+3YY=
+github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
+github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
 github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/varlink/go v0.0.0-20190502142041-0f1d566d194b/go.mod h1:YHaw8N660ESgMgLOZfLQqT1htFItynAUxMesFBho52s=
+github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g=
+github.com/vbauerster/mpb/v5 v5.3.0/go.mod h1:4yTkvAb8Cm4eylAp6t0JRq6pXDkFJ4krUlDqWYkakAs=
+github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
 github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
 github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb h1:zGWFAtiMcyryUHoUjUJX0/lt1H2+i2Ka2n+D3DImSNo=
-github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74=
 github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342 h1:083n9fJt2dWOpJd/X/q9Xgl5XtQLL22uSFYbzVqJssg=
-gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230119114711-6fe07bb33342/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
-gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438 h1:+qRai7XRl8omFQVCeHcaWzL542Yw64vfmuXG+79ZCIc=
-gitlab.com/nvidia/cloud-native/go-nvlib v0.0.0-20230209143738-95328d8c4438/go.mod h1:GStidGxhaqJhYFW1YpOnLvYCbL2EsM0od7IW4u7+JgU=
-golang.org/x/mod v0.5.0 h1:UG21uOlmZabA4fW5i7ZX6bjw1xELEGg/ZLgZq9auk/Q=
-golang.org/x/mod v0.5.0/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro=
-golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
+golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6 h1:cy1ko5847T/lJ45eyg/7uLprIE/amW5IXxGtEnQdYMI=
-golang.org/x/sys v0.0.0-20220927170352-d9d178bc13c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191001123449-8b695b21ef34/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
-gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
-gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
-sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A=
+k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA=
+k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
+k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
+k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
+k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

internal/config/cli.go

@@ -1,48 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package config
-
-import (
-	"github.com/pelletier/go-toml"
-)
-
-// ContainerCLIConfig stores the options for the nvidia-container-cli
-type ContainerCLIConfig struct {
-	Root string
-}
-
-// getContainerCLIConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getContainerCLIConfigFrom(toml *toml.Tree) *ContainerCLIConfig {
-	cfg := getDefaultContainerCLIConfig()
-
-	if toml == nil {
-		return cfg
-	}
-
-	cfg.Root = toml.GetDefault("nvidia-container-cli.root", cfg.Root).(string)
-
-	return cfg
-}
-
-// getDefaultContainerCLIConfig defines the default values for the config
-func getDefaultContainerCLIConfig() *ContainerCLIConfig {
-	c := ContainerCLIConfig{
-		Root: "",
-	}
-
-	return &c
-}

internal/config/config.go

@@ -1,126 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package config
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path"
-
-	"github.com/pelletier/go-toml"
-)
-
-const (
-	configOverride = "XDG_CONFIG_HOME"
-	configFilePath = "nvidia-container-runtime/config.toml"
-)
-
-var (
-	// DefaultExecutableDir specifies the default path to use for executables if they cannot be located in the path.
-	DefaultExecutableDir = "/usr/bin"
-
-	// NVIDIAContainerRuntimeHookExecutable is the executable name for the NVIDIA Container Runtime Hook
-	NVIDIAContainerRuntimeHookExecutable = "nvidia-container-runtime-hook"
-	// NVIDIAContainerToolkitExecutable is the executable name for the NVIDIA Container Toolkit (an alias for the NVIDIA Container Runtime Hook)
-	NVIDIAContainerToolkitExecutable = "nvidia-container-toolkit"
-
-	configDir = "/etc/"
-)
-
-// Config represents the contents of the config.toml file for the NVIDIA Container Toolkit
-// Note: This is currently duplicated by the HookConfig in cmd/nvidia-container-toolkit/hook_config.go
-type Config struct {
-	AcceptEnvvarUnprivileged bool `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
-
-	NVIDIAContainerCLIConfig         ContainerCLIConfig `toml:"nvidia-container-cli"`
-	NVIDIACTKConfig                  CTKConfig          `toml:"nvidia-ctk"`
-	NVIDIAContainerRuntimeConfig     RuntimeConfig      `toml:"nvidia-container-runtime"`
-	NVIDIAContainerRuntimeHookConfig RuntimeHookConfig  `toml:"nvidia-container-runtime-hook"`
-}
-
-// GetConfig sets up the config struct. Values are read from a toml file
-// or set via the environment.
-func GetConfig() (*Config, error) {
-	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
-		configDir = XDGConfigDir
-	}
-
-	configFilePath := path.Join(configDir, configFilePath)
-
-	tomlFile, err := os.Open(configFilePath)
-	if err != nil {
-		return getDefaultConfig(), nil
-	}
-	defer tomlFile.Close()
-
-	cfg, err := loadConfigFrom(tomlFile)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read config values: %v", err)
-	}
-
-	return cfg, nil
-}
-
-// loadRuntimeConfigFrom reads the config from the specified Reader
-func loadConfigFrom(reader io.Reader) (*Config, error) {
-	toml, err := toml.LoadReader(reader)
-	if err != nil {
-		return nil, err
-	}
-
-	return getConfigFrom(toml)
-}
-
-// getConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getConfigFrom(toml *toml.Tree) (*Config, error) {
-	cfg := getDefaultConfig()
-
-	if toml == nil {
-		return cfg, nil
-	}
-
-	cfg.AcceptEnvvarUnprivileged = toml.GetDefault("accept-nvidia-visible-devices-envvar-when-unprivileged", cfg.AcceptEnvvarUnprivileged).(bool)
-
-	cfg.NVIDIAContainerCLIConfig = *getContainerCLIConfigFrom(toml)
-	cfg.NVIDIACTKConfig = *getCTKConfigFrom(toml)
-	runtimeConfig, err := getRuntimeConfigFrom(toml)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load nvidia-container-runtime config: %v", err)
-	}
-	cfg.NVIDIAContainerRuntimeConfig = *runtimeConfig
-
-	runtimeHookConfig, err := getRuntimeHookConfigFrom(toml)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load nvidia-container-runtime-hook config: %v", err)
-	}
-	cfg.NVIDIAContainerRuntimeHookConfig = *runtimeHookConfig
-
-	return cfg, nil
-}
-
-// getDefaultConfig defines the default values for the config
-func getDefaultConfig() *Config {
-	c := Config{
-		AcceptEnvvarUnprivileged:     true,
-		NVIDIAContainerCLIConfig:     *getDefaultContainerCLIConfig(),
-		NVIDIACTKConfig:              *getDefaultCTKConfig(),
-		NVIDIAContainerRuntimeConfig: *GetDefaultRuntimeConfig(),
-	}
-
-	return &c
-}

internal/config/config_test.go

@@ -1,193 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package config
-
-import (
-	"io/ioutil"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestGetConfigWithCustomConfig(t *testing.T) {
-	wd, err := os.Getwd()
-	require.NoError(t, err)
-
-	// By default debug is disabled
-	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
-	testDir := filepath.Join(wd, "test")
-	filename := filepath.Join(testDir, configFilePath)
-
-	os.Setenv(configOverride, testDir)
-
-	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
-	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
-
-	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
-
-	cfg, err := GetConfig()
-	require.NoError(t, err)
-	require.Equal(t, cfg.NVIDIAContainerRuntimeConfig.DebugFilePath, "/nvidia-container-toolkit.log")
-}
-
-func TestGetConfig(t *testing.T) {
-	testCases := []struct {
-		description    string
-		contents       []string
-		expectedError  error
-		expectedConfig *Config
-	}{
-		{
-			description: "empty config is default",
-			expectedConfig: &Config{
-				AcceptEnvvarUnprivileged: true,
-				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "",
-				},
-				NVIDIAContainerRuntimeConfig: RuntimeConfig{
-					DebugFilePath: "/dev/null",
-					LogLevel:      "info",
-					Runtimes:      []string{"docker-runc", "runc"},
-					Mode:          "auto",
-					Modes: modesConfig{
-						CSV: csvModeConfig{
-							MountSpecPath: "/etc/nvidia-container-runtime/host-files-for-container.d",
-						},
-						CDI: cdiModeConfig{
-							DefaultKind:        "nvidia.com/gpu",
-							AnnotationPrefixes: []string{"cdi.k8s.io/"},
-						},
-					},
-				},
-				NVIDIACTKConfig: CTKConfig{
-					Path: "nvidia-ctk",
-				},
-			},
-		},
-		{
-			description: "config options set inline",
-			contents: []string{
-				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
-				"nvidia-container-cli.root = \"/bar/baz\"",
-				"nvidia-container-runtime.debug = \"/foo/bar\"",
-				"nvidia-container-runtime.experimental = true",
-				"nvidia-container-runtime.discover-mode = \"not-legacy\"",
-				"nvidia-container-runtime.log-level = \"debug\"",
-				"nvidia-container-runtime.runtimes = [\"/some/runtime\",]",
-				"nvidia-container-runtime.mode = \"not-auto\"",
-				"nvidia-container-runtime.modes.cdi.default-kind = \"example.vendor.com/device\"",
-				"nvidia-container-runtime.modes.cdi.annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
-				"nvidia-container-runtime.modes.csv.mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
-				"nvidia-ctk.path = \"/foo/bar/nvidia-ctk\"",
-			},
-			expectedConfig: &Config{
-				AcceptEnvvarUnprivileged: false,
-				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
-				},
-				NVIDIAContainerRuntimeConfig: RuntimeConfig{
-					DebugFilePath: "/foo/bar",
-					LogLevel:      "debug",
-					Runtimes:      []string{"/some/runtime"},
-					Mode:          "not-auto",
-					Modes: modesConfig{
-						CSV: csvModeConfig{
-							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
-						},
-						CDI: cdiModeConfig{
-							DefaultKind: "example.vendor.com/device",
-							AnnotationPrefixes: []string{
-								"cdi.k8s.io/",
-								"example.vendor.com/",
-							},
-						},
-					},
-				},
-				NVIDIACTKConfig: CTKConfig{
-					Path: "/foo/bar/nvidia-ctk",
-				},
-			},
-		},
-		{
-			description: "config options set in section",
-			contents: []string{
-				"accept-nvidia-visible-devices-envvar-when-unprivileged = false",
-				"[nvidia-container-cli]",
-				"root = \"/bar/baz\"",
-				"[nvidia-container-runtime]",
-				"debug = \"/foo/bar\"",
-				"experimental = true",
-				"discover-mode = \"not-legacy\"",
-				"log-level = \"debug\"",
-				"runtimes = [\"/some/runtime\",]",
-				"mode = \"not-auto\"",
-				"[nvidia-container-runtime.modes.cdi]",
-				"default-kind = \"example.vendor.com/device\"",
-				"annotation-prefixes = [\"cdi.k8s.io/\", \"example.vendor.com/\",]",
-				"[nvidia-container-runtime.modes.csv]",
-				"mount-spec-path = \"/not/etc/nvidia-container-runtime/host-files-for-container.d\"",
-				"[nvidia-ctk]",
-				"path = \"/foo/bar/nvidia-ctk\"",
-			},
-			expectedConfig: &Config{
-				AcceptEnvvarUnprivileged: false,
-				NVIDIAContainerCLIConfig: ContainerCLIConfig{
-					Root: "/bar/baz",
-				},
-				NVIDIAContainerRuntimeConfig: RuntimeConfig{
-					DebugFilePath: "/foo/bar",
-					LogLevel:      "debug",
-					Runtimes:      []string{"/some/runtime"},
-					Mode:          "not-auto",
-					Modes: modesConfig{
-						CSV: csvModeConfig{
-							MountSpecPath: "/not/etc/nvidia-container-runtime/host-files-for-container.d",
-						},
-						CDI: cdiModeConfig{
-							DefaultKind: "example.vendor.com/device",
-							AnnotationPrefixes: []string{
-								"cdi.k8s.io/",
-								"example.vendor.com/",
-							},
-						},
-					},
-				},
-				NVIDIACTKConfig: CTKConfig{
-					Path: "/foo/bar/nvidia-ctk",
-				},
-			},
-		},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.description, func(t *testing.T) {
-			reader := strings.NewReader(strings.Join(tc.contents, "\n"))
-
-			cfg, err := loadConfigFrom(reader)
-			if tc.expectedError != nil {
-				require.Error(t, err)
-			} else {
-				require.NoError(t, err)
-			}
-
-			require.EqualValues(t, tc.expectedConfig, cfg)
-		})
-	}
-}

internal/config/engine/api.go

@@ -1,25 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package engine
-
-// Interface defines the API for a runtime config updater.
-type Interface interface {
-	DefaultRuntime() string
-	AddRuntime(string, string, bool) error
-	RemoveRuntime(string) error
-	Save(string) (int64, error)
-}

internal/config/engine/containerd/config_v1.go

@@ -1,140 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package containerd
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
-	"github.com/pelletier/go-toml"
-)
-
-// ConfigV1 represents a version 1 containerd config
-type ConfigV1 Config
-
-var _ engine.Interface = (*ConfigV1)(nil)
-
-// AddRuntime adds a runtime to the containerd config
-func (c *ConfigV1) AddRuntime(name string, path string, setAsDefault bool) error {
-	if c == nil || c.Tree == nil {
-		return fmt.Errorf("config is nil")
-	}
-
-	config := *c.Tree
-
-	config.Set("version", int64(1))
-
-	switch runc := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", "runc"}).(type) {
-	case *toml.Tree:
-		runc, _ = toml.Load(runc.String())
-		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name}, runc)
-	}
-
-	if config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name}) == nil {
-		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
-		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_root"}, "")
-		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "runtime_engine"}, "")
-		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
-	}
-
-	if len(c.ContainerAnnotations) > 0 {
-		annotations, err := (*Config)(c).getRuntimeAnnotations([]string{"plugins", "cri", "containerd", "runtimes", name, "container_annotations"})
-		if err != nil {
-			return err
-		}
-		annotations = append(c.ContainerAnnotations, annotations...)
-		config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
-	}
-
-	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
-	config.SetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}, path)
-
-	if setAsDefault && c.UseDefaultRuntimeName {
-		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}, name)
-	} else if setAsDefault {
-		// Note: This is deprecated in containerd 1.4.0 and will be removed in 1.5.0
-		if config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"}) == nil {
-			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_type"}, c.RuntimeType)
-			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_root"}, "")
-			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "runtime_engine"}, "")
-			config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "privileged_without_host_devices"}, false)
-		}
-		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}, path)
-		config.SetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}, path)
-	}
-
-	*c.Tree = config
-	return nil
-}
-
-// DefaultRuntime returns the default runtime for the cri-o config
-func (c ConfigV1) DefaultRuntime() string {
-	if runtime, ok := c.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
-		return runtime
-	}
-	return ""
-}
-
-// RemoveRuntime removes a runtime from the docker config
-func (c *ConfigV1) RemoveRuntime(name string) error {
-	if c == nil || c.Tree == nil {
-		return nil
-	}
-
-	config := *c.Tree
-
-	// If the specified runtime was set as the default runtime we need to remove the default runtime too.
-	runtimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "BinaryName"}).(string)
-	if !ok || runtimePath == "" {
-		runtimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "runtimes", name, "options", "Runtime"}).(string)
-	}
-	defaultRuntimePath, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "BinaryName"}).(string)
-	if !ok || defaultRuntimePath == "" {
-		defaultRuntimePath, _ = config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime", "options", "Runtime"}).(string)
-	}
-	if runtimePath != "" && defaultRuntimePath != "" && runtimePath == defaultRuntimePath {
-		config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime"})
-	}
-
-	config.DeletePath([]string{"plugins", "cri", "containerd", "runtimes", name})
-	if runtime, ok := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"}).(string); ok {
-		if runtime == name {
-			config.DeletePath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
-		}
-	}
-
-	runtimeConfigPath := []string{"plugins", "cri", "containerd", "runtimes", name}
-	for i := 0; i < len(runtimeConfigPath); i++ {
-		if runtimes, ok := config.GetPath(runtimeConfigPath[:len(runtimeConfigPath)-i]).(*toml.Tree); ok {
-			if len(runtimes.Keys()) == 0 {
-				config.DeletePath(runtimeConfigPath[:len(runtimeConfigPath)-i])
-			}
-		}
-	}
-
-	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
-		config.Delete("version")
-	}
-
-	*c.Tree = config
-	return nil
-}
-
-// Save wrotes the config to a file
-func (c ConfigV1) Save(path string) (int64, error) {
-	return (Config)(c).Save(path)
-}

internal/config/engine/containerd/config_v2.go

@@ -1,161 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package containerd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/pelletier/go-toml"
-)
-
-// AddRuntime adds a runtime to the containerd config
-func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
-	if c == nil || c.Tree == nil {
-		return fmt.Errorf("config is nil")
-	}
-	config := *c.Tree
-
-	config.Set("version", int64(2))
-
-	switch runc := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", "runc"}).(type) {
-	case *toml.Tree:
-		runc, _ = toml.Load(runc.String())
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}, runc)
-	}
-
-	if config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}) == nil {
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_type"}, c.RuntimeType)
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_root"}, "")
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "runtime_engine"}, "")
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "privileged_without_host_devices"}, false)
-	}
-
-	if len(c.ContainerAnnotations) > 0 {
-		annotations, err := c.getRuntimeAnnotations([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"})
-		if err != nil {
-			return err
-		}
-		annotations = append(c.ContainerAnnotations, annotations...)
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "container_annotations"}, annotations)
-	}
-
-	config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name, "options", "BinaryName"}, path)
-
-	if setAsDefault {
-		config.SetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}, name)
-	}
-
-	*c.Tree = config
-	return nil
-}
-
-func (c *Config) getRuntimeAnnotations(path []string) ([]string, error) {
-	if c == nil || c.Tree == nil {
-		return nil, nil
-	}
-
-	config := *c.Tree
-	if !config.HasPath(path) {
-		return nil, nil
-	}
-	annotationsI, ok := config.GetPath(path).([]interface{})
-	if !ok {
-		return nil, fmt.Errorf("invalid annotations: %v", annotationsI)
-	}
-
-	var annotations []string
-	for _, annotation := range annotationsI {
-		a, ok := annotation.(string)
-		if !ok {
-			return nil, fmt.Errorf("invalid annotation: %v", annotation)
-		}
-		annotations = append(annotations, a)
-	}
-
-	return annotations, nil
-}
-
-// DefaultRuntime returns the default runtime for the cri-o config
-func (c Config) DefaultRuntime() string {
-	if runtime, ok := c.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
-		return runtime
-	}
-	return ""
-}
-
-// RemoveRuntime removes a runtime from the docker config
-func (c *Config) RemoveRuntime(name string) error {
-	if c == nil || c.Tree == nil {
-		return nil
-	}
-
-	config := *c.Tree
-
-	config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name})
-	if runtime, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"}).(string); ok {
-		if runtime == name {
-			config.DeletePath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
-		}
-	}
-
-	runtimePath := []string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes", name}
-	for i := 0; i < len(runtimePath); i++ {
-		if runtimes, ok := config.GetPath(runtimePath[:len(runtimePath)-i]).(*toml.Tree); ok {
-			if len(runtimes.Keys()) == 0 {
-				config.DeletePath(runtimePath[:len(runtimePath)-i])
-			}
-		}
-	}
-
-	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
-		config.Delete("version")
-	}
-
-	*c.Tree = config
-	return nil
-}
-
-// Save writes the config to the specified path
-func (c Config) Save(path string) (int64, error) {
-	config := c.Tree
-	output, err := config.ToTomlString()
-	if err != nil {
-		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
-	}
-
-	if len(output) == 0 {
-		err := os.Remove(path)
-		if err != nil {
-			return 0, fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		return 0, nil
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
-	}
-	defer f.Close()
-
-	n, err := f.WriteString(output)
-	if err != nil {
-		return 0, fmt.Errorf("unable to write output: %v", err)
-	}
-
-	return int64(n), err
-}

internal/config/engine/containerd/containerd.go

@@ -1,40 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package containerd
-
-import (
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
-	"github.com/pelletier/go-toml"
-)
-
-// Config represents the containerd config
-type Config struct {
-	*toml.Tree
-	RuntimeType           string
-	UseDefaultRuntimeName bool
-	ContainerAnnotations  []string
-}
-
-// New creates a containerd config with the specified options
-func New(opts ...Option) (engine.Interface, error) {
-	b := &builder{}
-	for _, opt := range opts {
-		opt(b)
-	}
-
-	return b.build()
-}

internal/config/engine/containerd/option.go

@@ -1,149 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package containerd
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
-	"github.com/pelletier/go-toml"
-	log "github.com/sirupsen/logrus"
-)
-
-const (
-	defaultRuntimeType = "io.containerd.runc.v2"
-)
-
-type builder struct {
-	path                 string
-	runtimeType          string
-	useLegacyConfig      bool
-	containerAnnotations []string
-}
-
-// Option defines a function that can be used to configure the config builder
-type Option func(*builder)
-
-// WithPath sets the path for the config builder
-func WithPath(path string) Option {
-	return func(b *builder) {
-		b.path = path
-	}
-}
-
-// WithRuntimeType sets the runtime type for the config builder
-func WithRuntimeType(runtimeType string) Option {
-	return func(b *builder) {
-		b.runtimeType = runtimeType
-	}
-}
-
-// WithUseLegacyConfig sets the useLegacyConfig flag for the config builder
-func WithUseLegacyConfig(useLegacyConfig bool) Option {
-	return func(b *builder) {
-		b.useLegacyConfig = useLegacyConfig
-	}
-}
-
-// WithContainerAnnotations sets the container annotations for the config builder
-func WithContainerAnnotations(containerAnnotations ...string) Option {
-	return func(b *builder) {
-		b.containerAnnotations = containerAnnotations
-	}
-}
-
-func (b *builder) build() (engine.Interface, error) {
-	if b.path == "" {
-		return nil, fmt.Errorf("config path is empty")
-	}
-
-	if b.runtimeType == "" {
-		b.runtimeType = defaultRuntimeType
-	}
-
-	config, err := loadConfig(b.path)
-	if err != nil {
-		return nil, fmt.Errorf("failed to load config: %v", err)
-	}
-	config.RuntimeType = b.runtimeType
-	config.UseDefaultRuntimeName = !b.useLegacyConfig
-	config.ContainerAnnotations = b.containerAnnotations
-
-	version, err := config.parseVersion(b.useLegacyConfig)
-	if err != nil {
-		return nil, fmt.Errorf("failed to parse config version: %v", err)
-	}
-	switch version {
-	case 1:
-		return (*ConfigV1)(config), nil
-	case 2:
-		return config, nil
-	}
-
-	return nil, fmt.Errorf("unsupported config version: %v", version)
-}
-
-// loadConfig loads the containerd config from disk
-func loadConfig(config string) (*Config, error) {
-	log.Infof("Loading config: %v", config)
-
-	info, err := os.Stat(config)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	configFile := config
-	if os.IsNotExist(err) {
-		configFile = "/dev/null"
-		log.Infof("Config file does not exist, creating new one")
-	}
-
-	tomlConfig, err := toml.LoadFile(configFile)
-	if err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-
-	cfg := Config{
-		Tree: tomlConfig,
-	}
-	return &cfg, nil
-}
-
-// parseVersion returns the version of the config
-func (c *Config) parseVersion(useLegacyConfig bool) (int, error) {
-	defaultVersion := 2
-	if useLegacyConfig {
-		defaultVersion = 1
-	}
-
-	switch v := c.Get("version").(type) {
-	case nil:
-		switch len(c.Keys()) {
-		case 0: // No config exists, or the config file is empty, use version inferred from containerd
-			return defaultVersion, nil
-		default: // A config file exists, has content, and no version is set
-			return 1, nil
-		}
-	case int64:
-		return int(v), nil
-	default:
-		return -1, fmt.Errorf("unsupported type for version field: %v", v)
-	}
-}

internal/config/engine/crio/crio.go

@@ -1,131 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package crio
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
-	"github.com/pelletier/go-toml"
-)
-
-// Config represents the cri-o config
-type Config toml.Tree
-
-// New creates a cri-o config with the specified options
-func New(opts ...Option) (engine.Interface, error) {
-	b := &builder{}
-	for _, opt := range opts {
-		opt(b)
-	}
-
-	return b.build()
-}
-
-// AddRuntime adds a new runtime to the crio config
-func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
-	if c == nil {
-		return fmt.Errorf("config is nil")
-	}
-
-	config := (toml.Tree)(*c)
-
-	switch runc := config.Get("crio.runtime.runtimes.runc").(type) {
-	case *toml.Tree:
-		runc, _ = toml.Load(runc.String())
-		config.SetPath([]string{"crio", "runtime", "runtimes", name}, runc)
-	}
-
-	config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_path"}, path)
-	config.SetPath([]string{"crio", "runtime", "runtimes", name, "runtime_type"}, "oci")
-
-	if setAsDefault {
-		config.SetPath([]string{"crio", "runtime", "default_runtime"}, name)
-	}
-
-	*c = (Config)(config)
-	return nil
-}
-
-// DefaultRuntime returns the default runtime for the cri-o config
-func (c Config) DefaultRuntime() string {
-	config := (toml.Tree)(c)
-	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
-		return runtime
-	}
-	return ""
-}
-
-// RemoveRuntime removes a runtime from the cri-o config
-func (c *Config) RemoveRuntime(name string) error {
-	if c == nil {
-		return nil
-	}
-
-	config := (toml.Tree)(*c)
-	if runtime, ok := config.GetPath([]string{"crio", "runtime", "default_runtime"}).(string); ok {
-		if runtime == name {
-			config.DeletePath([]string{"crio", "runtime", "default_runtime"})
-		}
-	}
-
-	runtimeClassPath := []string{"crio", "runtime", "runtimes", name}
-	config.DeletePath(runtimeClassPath)
-	for i := 0; i < len(runtimeClassPath); i++ {
-		remainingPath := runtimeClassPath[:len(runtimeClassPath)-i]
-		if entry, ok := config.GetPath(remainingPath).(*toml.Tree); ok {
-			if len(entry.Keys()) != 0 {
-				break
-			}
-			config.DeletePath(remainingPath)
-		}
-	}
-
-	*c = (Config)(config)
-	return nil
-}
-
-// Save writes the config to the specified path
-func (c Config) Save(path string) (int64, error) {
-	config := (toml.Tree)(c)
-	output, err := config.ToTomlString()
-	if err != nil {
-		return 0, fmt.Errorf("unable to convert to TOML: %v", err)
-	}
-
-	if len(output) == 0 {
-		err := os.Remove(path)
-		if err != nil {
-			return 0, fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		return 0, nil
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return 0, fmt.Errorf("unable to open '%v' for writing: %v", path, err)
-	}
-	defer f.Close()
-
-	n, err := f.WriteString(output)
-	if err != nil {
-		return 0, fmt.Errorf("unable to write output: %v", err)
-	}
-
-	return int64(n), err
-}

internal/config/engine/crio/option.go

@@ -1,73 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package crio
-
-import (
-	"fmt"
-	"os"
-
-	"github.com/pelletier/go-toml"
-	log "github.com/sirupsen/logrus"
-)
-
-type builder struct {
-	path string
-}
-
-// Option defines a function that can be used to configure the config builder
-type Option func(*builder)
-
-// WithPath sets the path for the config builder
-func WithPath(path string) Option {
-	return func(b *builder) {
-		b.path = path
-	}
-}
-
-func (b *builder) build() (*Config, error) {
-	if b.path == "" {
-		empty := toml.Tree{}
-		return (*Config)(&empty), nil
-	}
-
-	return loadConfig(b.path)
-}
-
-// loadConfig loads the cri-o config from disk
-func loadConfig(config string) (*Config, error) {
-	log.Infof("Loading config: %v", config)
-
-	info, err := os.Stat(config)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	configFile := config
-	if os.IsNotExist(err) {
-		configFile = "/dev/null"
-		log.Infof("Config file does not exist, creating new one")
-	}
-
-	cfg, err := toml.LoadFile(configFile)
-	if err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-
-	return (*Config)(cfg), nil
-}

internal/config/engine/docker/docker.go

@@ -1,140 +0,0 @@
-/**
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package docker
-
-import (
-	"encoding/json"
-	"fmt"
-	"os"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/engine"
-)
-
-const (
-	defaultDockerRuntime = "runc"
-)
-
-// Config defines a docker config file.
-// TODO: This should not be public, but we need to access it from the tests in tools/container/docker
-type Config map[string]interface{}
-
-// New creates a docker config with the specified options
-func New(opts ...Option) (engine.Interface, error) {
-	b := &builder{}
-	for _, opt := range opts {
-		opt(b)
-	}
-
-	return b.build()
-}
-
-// AddRuntime adds a new runtime to the docker config
-func (c *Config) AddRuntime(name string, path string, setAsDefault bool) error {
-	if c == nil {
-		return fmt.Errorf("config is nil")
-	}
-
-	config := *c
-
-	// Read the existing runtimes
-	runtimes := make(map[string]interface{})
-	if _, exists := config["runtimes"]; exists {
-		runtimes = config["runtimes"].(map[string]interface{})
-	}
-
-	// Add / update the runtime definitions
-	runtimes[name] = map[string]interface{}{
-		"path": path,
-		"args": []string{},
-	}
-
-	config["runtimes"] = runtimes
-
-	if setAsDefault {
-		config["default-runtime"] = name
-	}
-
-	*c = config
-	return nil
-}
-
-// DefaultRuntime returns the default runtime for the docker config
-func (c Config) DefaultRuntime() string {
-	r, ok := c["default-runtime"].(string)
-	if !ok {
-		return ""
-	}
-	return r
-}
-
-// RemoveRuntime removes a runtime from the docker config
-func (c *Config) RemoveRuntime(name string) error {
-	if c == nil {
-		return nil
-	}
-	config := *c
-
-	if _, exists := config["default-runtime"]; exists {
-		defaultRuntime := config["default-runtime"].(string)
-		if defaultRuntime == name {
-			config["default-runtime"] = defaultDockerRuntime
-		}
-	}
-
-	if _, exists := config["runtimes"]; exists {
-		runtimes := config["runtimes"].(map[string]interface{})
-
-		delete(runtimes, name)
-
-		if len(runtimes) == 0 {
-			delete(config, "runtimes")
-		}
-	}
-
-	*c = config
-
-	return nil
-}
-
-// Save writes the config to the specified path
-func (c Config) Save(path string) (int64, error) {
-	output, err := json.MarshalIndent(c, "", "    ")
-	if err != nil {
-		return 0, fmt.Errorf("unable to convert to JSON: %v", err)
-	}
-
-	if len(output) == 0 {
-		err := os.Remove(path)
-		if err != nil {
-			return 0, fmt.Errorf("unable to remove empty file: %v", err)
-		}
-		return 0, nil
-	}
-
-	f, err := os.Create(path)
-	if err != nil {
-		return 0, fmt.Errorf("unable to open %v for writing: %v", path, err)
-	}
-	defer f.Close()
-
-	n, err := f.WriteString(string(output))
-	if err != nil {
-		return 0, fmt.Errorf("unable to write output: %v", err)
-	}
-
-	return int64(n), nil
-}

internal/config/engine/docker/docker_test.go

@@ -1,215 +0,0 @@
-/**
-# Copyright (c) 2021-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package docker
-
-import (
-	"encoding/json"
-	"fmt"
-	"testing"
-
-	"github.com/stretchr/testify/require"
-)
-
-func TestUpdateConfigDefaultRuntime(t *testing.T) {
-	testCases := []struct {
-		config                     Config
-		runtimeName                string
-		setAsDefault               bool
-		expectedDefaultRuntimeName interface{}
-	}{
-		{
-			setAsDefault:               false,
-			expectedDefaultRuntimeName: nil,
-		},
-		{
-			runtimeName:                "NAME",
-			setAsDefault:               true,
-			expectedDefaultRuntimeName: "NAME",
-		},
-		{
-			config: map[string]interface{}{
-				"default-runtime": "ALREADY_SET",
-			},
-			runtimeName:                "NAME",
-			setAsDefault:               false,
-			expectedDefaultRuntimeName: "ALREADY_SET",
-		},
-		{
-			config: map[string]interface{}{
-				"default-runtime": "ALREADY_SET",
-			},
-			runtimeName:                "NAME",
-			setAsDefault:               true,
-			expectedDefaultRuntimeName: "NAME",
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			if tc.config == nil {
-				tc.config = make(map[string]interface{})
-			}
-			err := tc.config.AddRuntime(tc.runtimeName, "", tc.setAsDefault)
-			require.NoError(t, err)
-
-			defaultRuntimeName := tc.config["default-runtime"]
-			require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName)
-		})
-	}
-}
-
-func TestUpdateConfigRuntimes(t *testing.T) {
-	testCases := []struct {
-		config         Config
-		runtimes       map[string]string
-		expectedConfig map[string]interface{}
-	}{
-		{
-			config: map[string]interface{}{},
-			runtimes: map[string]string{
-				"runtime1": "/test/runtime/dir/runtime1",
-				"runtime2": "/test/runtime/dir/runtime2",
-			},
-			expectedConfig: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"runtime1": map[string]interface{}{
-						"path": "/test/runtime/dir/runtime1",
-						"args": []string{},
-					},
-					"runtime2": map[string]interface{}{
-						"path": "/test/runtime/dir/runtime2",
-						"args": []string{},
-					},
-				},
-			},
-		},
-		{
-			config: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"runtime1": map[string]interface{}{
-						"path": "runtime1",
-						"args": []string{},
-					},
-				},
-			},
-			runtimes: map[string]string{
-				"runtime1": "/test/runtime/dir/runtime1",
-				"runtime2": "/test/runtime/dir/runtime2",
-			},
-			expectedConfig: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"runtime1": map[string]interface{}{
-						"path": "/test/runtime/dir/runtime1",
-						"args": []string{},
-					},
-					"runtime2": map[string]interface{}{
-						"path": "/test/runtime/dir/runtime2",
-						"args": []string{},
-					},
-				},
-			},
-		},
-		{
-			config: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"not-nvidia": map[string]interface{}{
-						"path": "some-other-path",
-						"args": []string{},
-					},
-				},
-			},
-			runtimes: map[string]string{
-				"runtime1": "/test/runtime/dir/runtime1",
-			},
-			expectedConfig: map[string]interface{}{
-				"runtimes": map[string]interface{}{
-					"not-nvidia": map[string]interface{}{
-						"path": "some-other-path",
-						"args": []string{},
-					},
-					"runtime1": map[string]interface{}{
-						"path": "/test/runtime/dir/runtime1",
-						"args": []string{},
-					},
-				},
-			},
-		},
-		{
-			config: map[string]interface{}{
-				"exec-opts":  []string{"native.cgroupdriver=systemd"},
-				"log-driver": "json-file",
-				"log-opts": map[string]string{
-					"max-size": "100m",
-				},
-				"storage-driver": "overlay2",
-			},
-			runtimes: map[string]string{
-				"runtime1": "/test/runtime/dir/runtime1",
-			},
-			expectedConfig: map[string]interface{}{
-				"exec-opts":  []string{"native.cgroupdriver=systemd"},
-				"log-driver": "json-file",
-				"log-opts": map[string]string{
-					"max-size": "100m",
-				},
-				"storage-driver": "overlay2",
-				"runtimes": map[string]interface{}{
-					"runtime1": map[string]interface{}{
-						"path": "/test/runtime/dir/runtime1",
-						"args": []string{},
-					},
-				},
-			},
-		},
-		{
-			config: map[string]interface{}{
-				"exec-opts":  []string{"native.cgroupdriver=systemd"},
-				"log-driver": "json-file",
-				"log-opts": map[string]string{
-					"max-size": "100m",
-				},
-				"storage-driver": "overlay2",
-			},
-			expectedConfig: map[string]interface{}{
-				"exec-opts":  []string{"native.cgroupdriver=systemd"},
-				"log-driver": "json-file",
-				"log-opts": map[string]string{
-					"max-size": "100m",
-				},
-				"storage-driver": "overlay2",
-			},
-		},
-	}
-
-	for i, tc := range testCases {
-		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
-			for runtimeName, runtimePath := range tc.runtimes {
-				err := tc.config.AddRuntime(runtimeName, runtimePath, false)
-				require.NoError(t, err)
-			}
-
-			configContent, err := json.MarshalIndent(tc.config, "", "    ")
-			require.NoError(t, err)
-
-			expectedContent, err := json.MarshalIndent(tc.expectedConfig, "", "    ")
-			require.NoError(t, err)
-
-			require.EqualValues(t, string(expectedContent), string(configContent))
-		})
-
-	}
-}

internal/config/engine/docker/option.go

@@ -1,80 +0,0 @@
-/**
-# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package docker
-
-import (
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
-	"os"
-
-	log "github.com/sirupsen/logrus"
-)
-
-type builder struct {
-	path string
-}
-
-// Option defines a function that can be used to configure the config builder
-type Option func(*builder)
-
-// WithPath sets the path for the config builder
-func WithPath(path string) Option {
-	return func(b *builder) {
-		b.path = path
-	}
-}
-
-func (b *builder) build() (*Config, error) {
-	if b.path == "" {
-		empty := make(Config)
-		return &empty, nil
-	}
-
-	return loadConfig(b.path)
-}
-
-// loadConfig loads the docker config from disk
-func loadConfig(configFilePath string) (*Config, error) {
-	log.Infof("Loading docker config from %v", configFilePath)
-
-	info, err := os.Stat(configFilePath)
-	if os.IsExist(err) && info.IsDir() {
-		return nil, fmt.Errorf("config file is a directory")
-	}
-
-	cfg := make(Config)
-
-	if os.IsNotExist(err) {
-		log.Infof("Config file does not exist, creating new one")
-		return &cfg, nil
-	}
-
-	readBytes, err := ioutil.ReadFile(configFilePath)
-	if err != nil {
-		return nil, fmt.Errorf("unable to read config: %v", err)
-	}
-
-	reader := bytes.NewReader(readBytes)
-	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
-		return nil, err
-	}
-
-	log.Infof("Successfully loaded config")
-	return &cfg, nil
-}

internal/config/hook.go

@@ -1,62 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package config
-
-import (
-	"fmt"
-
-	"github.com/pelletier/go-toml"
-)
-
-// RuntimeHookConfig stores the config options for the NVIDIA Container Runtime
-type RuntimeHookConfig struct {
-	// SkipModeDetection disables the mode check for the runtime hook.
-	SkipModeDetection bool `toml:"skip-mode-detection"`
-}
-
-// dummyHookConfig allows us to unmarshal only a RuntimeHookConfig from a *toml.Tree
-type dummyHookConfig struct {
-	RuntimeHook RuntimeHookConfig `toml:"nvidia-container-runtime-hook"`
-}
-
-// getRuntimeHookConfigFrom reads the nvidia container runtime config from the specified toml Tree.
-func getRuntimeHookConfigFrom(toml *toml.Tree) (*RuntimeHookConfig, error) {
-	cfg := GetDefaultRuntimeHookConfig()
-
-	if toml == nil {
-		return cfg, nil
-	}
-
-	d := dummyHookConfig{
-		RuntimeHook: *cfg,
-	}
-
-	if err := toml.Unmarshal(&d); err != nil {
-		return nil, fmt.Errorf("failed to unmarshal runtime config: %v", err)
-	}
-
-	return &d.RuntimeHook, nil
-}
-
-// GetDefaultRuntimeHookConfig defines the default values for the config
-func GetDefaultRuntimeHookConfig() *RuntimeHookConfig {
-	c := RuntimeHookConfig{
-		SkipModeDetection: false,
-	}
-
-	return &c
-}

internal/config/image/capabilities.go

@@ -1,54 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package image
-
-// DriverCapability represents the possible values of NVIDIA_DRIVER_CAPABILITIES
-type DriverCapability string
-
-// Constants for the supported driver capabilities
-const (
-	DriverCapabilityAll      DriverCapability = "all"
-	DriverCapabilityCompat32 DriverCapability = "compat32"
-	DriverCapabilityCompute  DriverCapability = "compute"
-	DriverCapabilityDisplay  DriverCapability = "display"
-	DriverCapabilityGraphics DriverCapability = "graphics"
-	DriverCapabilityNgx      DriverCapability = "ngx"
-	DriverCapabilityUtility  DriverCapability = "utility"
-	DriverCapabilityVideo    DriverCapability = "video"
-)
-
-// DriverCapabilities represents the NVIDIA_DRIVER_CAPABILITIES set for the specified image.
-type DriverCapabilities map[DriverCapability]bool
-
-// Has check whether the specified capability is selected.
-func (c DriverCapabilities) Has(capability DriverCapability) bool {
-	if c[DriverCapabilityAll] {
-		return true
-	}
-	return c[capability]
-}
-
-// Any checks whether any of the specified capabilites are set
-func (c DriverCapabilities) Any(capabilities ...DriverCapability) bool {
-	for _, cap := range capabilities {
-		if c.Has(cap) {
-			return true
-		}
-	}
-
-	return false
-}

internal/config/image/cuda_image.go

@@ -1,190 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package image
-
-import (
-	"fmt"
-	"strconv"
-	"strings"
-
-	"github.com/opencontainers/runtime-spec/specs-go"
-	"golang.org/x/mod/semver"
-)
-
-const (
-	envCUDAVersion          = "CUDA_VERSION"
-	envNVRequirePrefix      = "NVIDIA_REQUIRE_"
-	envNVRequireCUDA        = envNVRequirePrefix + "CUDA"
-	envNVRequireJetpack     = envNVRequirePrefix + "JETPACK"
-	envNVDisableRequire     = "NVIDIA_DISABLE_REQUIRE"
-	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
-)
-
-// CUDA represents a CUDA image that can be used for GPU computing. This wraps
-// a map of environment variable to values that can be used to perform lookups
-// such as requirements.
-type CUDA map[string]string
-
-// NewCUDAImageFromSpec creates a CUDA image from the input OCI runtime spec.
-// The process environment is read (if present) to construc the CUDA Image.
-func NewCUDAImageFromSpec(spec *specs.Spec) (CUDA, error) {
-	if spec == nil || spec.Process == nil {
-		return NewCUDAImageFromEnv(nil)
-	}
-
-	return NewCUDAImageFromEnv(spec.Process.Env)
-}
-
-// NewCUDAImageFromEnv creates a CUDA image from the input environment. The environment
-// is a list of strings of the form ENVAR=VALUE.
-func NewCUDAImageFromEnv(env []string) (CUDA, error) {
-	c := make(CUDA)
-
-	for _, e := range env {
-		parts := strings.SplitN(e, "=", 2)
-		if len(parts) != 2 {
-			return nil, fmt.Errorf("invalid environment variable: %v", e)
-		}
-		c[parts[0]] = parts[1]
-	}
-
-	return c, nil
-}
-
-// IsLegacy returns whether the associated CUDA image is a "legacy" image. An
-// image is considered legacy if it has a CUDA_VERSION environment variable defined
-// and no NVIDIA_REQUIRE_CUDA environment variable defined.
-func (i CUDA) IsLegacy() bool {
-	legacyCudaVersion := i[envCUDAVersion]
-	cudaRequire := i[envNVRequireCUDA]
-	return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0
-}
-
-// GetRequirements returns the requirements from all NVIDIA_REQUIRE_ environment
-// variables.
-func (i CUDA) GetRequirements() ([]string, error) {
-	// TODO: We need not process this if disable require is set, but this will be done
-	// in a single follow-up to ensure that the behavioural change is accurately captured.
-	// if i.HasDisableRequire() {
-	// 	return nil, nil
-	// }
-
-	// All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli
-	var requirements []string
-	for name, value := range i {
-		if strings.HasPrefix(name, envNVRequirePrefix) && !strings.HasPrefix(name, envNVRequireJetpack) {
-			requirements = append(requirements, value)
-		}
-	}
-	if i.IsLegacy() {
-		v, err := i.legacyVersion()
-		if err != nil {
-			return nil, fmt.Errorf("failed to get version: %v", err)
-		}
-		cudaRequire := fmt.Sprintf("cuda>=%s", v)
-		requirements = append(requirements, cudaRequire)
-	}
-	return requirements, nil
-}
-
-// HasDisableRequire checks for the value of the NVIDIA_DISABLE_REQUIRE. If set
-// to a valid (true) boolean value this can be used to disable the requirement checks
-func (i CUDA) HasDisableRequire() bool {
-	if disable, exists := i[envNVDisableRequire]; exists {
-		// i.logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", disable)
-		d, _ := strconv.ParseBool(disable)
-		return d
-	}
-
-	return false
-}
-
-// DevicesFromEnvvars returns the devices requested by the image through environment variables
-func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
-	// We concantenate all the devices from the specified envvars.
-	var isSet bool
-	var devices []string
-	requested := make(map[string]bool)
-	for _, envVar := range envVars {
-		if devs, ok := i[envVar]; ok {
-			isSet = true
-			for _, d := range strings.Split(devs, ",") {
-				trimmed := strings.TrimSpace(d)
-				if len(trimmed) == 0 {
-					continue
-				}
-				devices = append(devices, trimmed)
-				requested[trimmed] = true
-			}
-		}
-	}
-
-	// Environment variable unset with legacy image: default to "all".
-	if !isSet && len(devices) == 0 && i.IsLegacy() {
-		return NewVisibleDevices("all")
-	}
-
-	// Environment variable unset or empty or "void": return nil
-	if len(devices) == 0 || requested["void"] {
-		return NewVisibleDevices("void")
-	}
-
-	return NewVisibleDevices(devices...)
-}
-
-// GetDriverCapabilities returns the requested driver capabilities.
-func (i CUDA) GetDriverCapabilities() DriverCapabilities {
-	env := i[envNVDriverCapabilities]
-
-	capabilites := make(DriverCapabilities)
-	for _, c := range strings.Split(env, ",") {
-		capabilites[DriverCapability(c)] = true
-	}
-
-	return capabilites
-}
-
-func (i CUDA) legacyVersion() (string, error) {
-	majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion])
-	if err != nil {
-		return "", fmt.Errorf("invalid CUDA version: %v", err)
-	}
-
-	return majorMinor, nil
-}
-
-func parseMajorMinorVersion(version string) (string, error) {
-	vVersion := "v" + strings.TrimPrefix(version, "v")
-
-	if !semver.IsValid(vVersion) {
-		return "", fmt.Errorf("invalid version string")
-	}
-
-	majorMinor := strings.TrimPrefix(semver.MajorMinor(vVersion), "v")
-	parts := strings.Split(majorMinor, ".")
-
-	var err error
-	_, err = strconv.ParseUint(parts[0], 10, 32)
-	if err != nil {
-		return "", fmt.Errorf("invalid major version")
-	}
-	_, err = strconv.ParseUint(parts[1], 10, 32)
-	if err != nil {
-		return "", fmt.Errorf("invalid minor version")
-	}
-	return majorMinor, nil
-}