Merge branch 'update-libnvidia-container' into 'master'

Update libnvidia-container to latest for release

See merge request nvidia/container-toolkit/container-toolkit!83

.common-ci.yml

@@ -0,0 +1,409 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+default:
+  image: docker:stable
+  services:
+    - name: docker:stable-dind
+      command: ["--experimental"]
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+  BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}"
+
+stages:
+  - image
+  - lint
+  - go-checks
+  - go-build
+  - unit-tests
+  - package-build
+  - image-build
+  - test
+  - scan
+  - release
+  - build-all
+
+build-dev-image:
+  stage: image
+  script:
+    - apk --no-cache add make bash
+    - make .build-image
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - make .push-build-image
+
+.requires-build-image:
+  image: "${BUILDIMAGE}"
+
+.go-check:
+  extends:
+    - .requires-build-image
+  stage: go-checks
+
+fmt:
+  extends:
+    - .go-check
+  script:
+    - make assert-fmt
+
+vet:
+  extends:
+    - .go-check
+  script:
+    - make vet
+
+lint:
+  extends:
+    - .go-check
+  script:
+    - make lint
+  allow_failure: true
+
+ineffassign:
+  extends:
+    - .go-check
+  script:
+    - make ineffassign
+  allow_failure: true
+
+misspell:
+  extends:
+    - .go-check
+  script:
+    - make misspell
+
+go-build:
+  extends:
+    - .requires-build-image
+  stage: go-build
+  script:
+    - make build
+
+unit-tests:
+  extends:
+    - .requires-build-image
+  stage: unit-tests
+  script:
+    - make coverage
+
+
+# Define the distribution targets
+.dist-centos7:
+  variables:
+    DIST: centos7
+
+.dist-centos8:
+  variables:
+    DIST: centos8
+
+.dist-ubi8:
+  variables:
+    DIST: ubi8
+
+.dist-ubuntu18.04:
+  variables:
+    DIST: ubuntu18.04
+
+.arch-aarch64:
+  variables:
+    ARCH: aarch64
+
+.arch-amd64:
+  variables:
+    ARCH: amd64
+
+.arch-arm64:
+  variables:
+    ARCH: arm64
+
+.arch-ppc64le:
+  variables:
+    ARCH: ppc64le
+
+.arch-x86_64:
+  variables:
+    ARCH: x86_64
+
+# Define the package build helpers
+.multi-arch-build:
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
+
+.package-artifacts:
+  variables:
+    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
+    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
+    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
+
+.package-build:
+  extends:
+    - .multi-arch-build
+    - .package-artifacts
+  stage: package-build
+  script:
+    - ./scripts/release.sh ${DIST}-${ARCH}
+
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
+
+# Define the package build targets
+package-ubuntu18.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-amd64
+
+package-ubuntu18.04-arm64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-arm64
+
+package-ubuntu18.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-ppc64le
+
+package-centos7-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-x86_64
+
+package-centos8-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-x86_64
+
+# Define the image build targets
+.image-build:
+  stage: image-build
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache bash make
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+
+image-centos7:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos7
+  needs:
+    - package-centos7-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-centos8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos8
+  needs:
+    - package-centos8-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-ubi8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubi8
+  needs:
+    # Note: The ubi8 image currently uses the centos7 packages
+    - package-centos7-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-ubuntu18.04:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubuntu18.04
+  needs:
+    - package-ubuntu18.04-amd64
+    # TODO: These will be required once we generate multi-arch images
+    # - package-ubuntu18.04-arm64
+    # - package-ubuntu18.04-ppc64le
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+# Define test helpers
+.integration:
+  stage: test
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache make bash jq
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
+  script:
+    - make -f build/container/Makefile test-${DIST}
+
+.test:toolkit:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "toolkit"
+
+.test:docker:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "docker"
+
+.test:containerd:
+  # TODO: The containerd tests fail due to issues with SIGHUP.
+  # Until this is resolved with retry up to twice and allow failure here.
+  retry: 2
+  allow_failure: true
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "containerd"
+
+.test:crio:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "crio"
+
+# Define the test targets
+test-toolkit-ubuntu18.04:
+  extends:
+    - .test:toolkit
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-containerd-ubuntu18.04:
+  extends:
+    - .test:containerd
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-crio-ubuntu18.04:
+  extends:
+    - .test:crio
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-docker-ubuntu18.04:
+  extends:
+    - .test:docker
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+# .release forms the base of the deployment jobs which push images to the CI registry.
+# This is extended with the version to be deployed (e.g. the SHA or TAG) and the
+# target os.
+.release:
+  stage:
+    release
+  variables:
+    # Define the source image for the release
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    # OUT_IMAGE_VERSION is overridden for external releases
+    OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
+  stage: release
+  before_script:
+    # We ensure that the OUT_IMAGE_VERSION is set
+    - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
+
+    # In the case where we are deploying a different version to the CI_COMMIT_SHA, we
+    # need to tag the image.
+    # Note: a leading 'v' is stripped from the version if present
+    - apk add --no-cache make bash
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
+  script:
+    - docker tag "${IMAGE_NAME}:${VERSION}-${DIST}" "${OUT_IMAGE_NAME}:${OUT_IMAGE_VERSION}-${DIST}"
+    # Log in to the "output" registry, tag the image and push the image
+    - 'echo "Logging in to output registry ${OUT_REGISTRY}"'
+    - docker logout
+    - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}"
+    - make IMAGE_NAME=${OUT_IMAGE_NAME} VERSION=${OUT_IMAGE_VERSION} -f build/container/Makefile push-${DIST}
+
+# Define a staging release step that pushes an image to an internal "staging" repository
+# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
+# outside of the release process.
+.release:staging:
+  extends:
+    - .release
+  variables:
+    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    OUT_REGISTRY: "${CI_REGISTRY}"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit"
+
+# Define an external release step that pushes an image to an external repository.
+# This includes a devlopment image off master.
+.release:external:
+  extends:
+    - .release
+  rules:
+    - if: $CI_COMMIT_TAG
+      variables:
+        OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}"
+    - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH
+      variables:
+        OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
+
+# Define the release jobs
+release:staging-centos7:
+  extends:
+    - .release:staging
+    - .dist-centos7
+  needs:
+    - image-centos7
+
+release:staging-centos8:
+  extends:
+    - .release:staging
+    - .dist-centos8
+  needs:
+    - image-centos8
+
+release:staging-ubi8:
+  extends:
+    - .release:staging
+    - .dist-ubi8
+  needs:
+    - image-ubi8
+
+release:staging-ubuntu18.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu18.04
+  needs:
+    - test-toolkit-ubuntu18.04
+    - test-containerd-ubuntu18.04
+    - test-crio-ubuntu18.04
+    - test-docker-ubuntu18.04

.dockerignore

@@ -1,2 +1,2 @@
 .git
-dist
+/shared-*

.gitignore

@@ -1,3 +1,8 @@
 dist
 *.swp
 *.swo
+/coverage.out
+/test/output/
+/nvidia-container-runtime
+/nvidia-container-toolkit
+/shared-*

.gitlab-ci.yml

@@ -1,161 +1,60 @@
-# Build packages for all supported OS / ARCH combinations
-
-stages:
-  - tests
-  - build-one
-  - build-all
-
-.tests-setup: &tests-setup
-  image: golang:1.14.4
-
-  rules:
-    - when: always
-
-  variables:
-    GITHUB_ROOT: "github.com/NVIDIA"
-    PROJECT_GOPATH: "${GITHUB_ROOT}/nvidia-container-toolkit"
-
-  before_script:
-    - mkdir -p ${GOPATH}/src/${GITHUB_ROOT}
-    - ln -s ${CI_PROJECT_DIR} ${GOPATH}/src/${PROJECT_GOPATH}
-
-.build-setup: &build-setup
-  image: docker:19.03.8
-
-  services:
-    - name: docker:19.03.8-dind
-      command: ["--experimental"]
-
-  before_script:
-  - apk update
-  - apk upgrade
-  - apk add coreutils build-base sed git bash make
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes
-
-# Run a series of sanity-check tests over the code
-lint:
-  <<: *tests-setup
-  stage: tests
-  script:
-    - go get -u golang.org/x/lint/golint
-    - golint -set_exit_status ${PROJECT_GOPATH}/pkg
-
-vet:
-  <<: *tests-setup
-  stage: tests
-  script:
-    - go vet ${PROJECT_GOPATH}/pkg
-
-unit_test:
-  <<: *tests-setup
-  stage: tests
-  script:
-    - go test ${PROJECT_GOPATH}/pkg
-
-fmt:
-  <<: *tests-setup
-  stage: tests
-  script:
-    - res=$(gofmt -l pkg/*.go)
-    - echo "$res"
-    - test -z "$res"
-
-ineffassign:
-  <<: *tests-setup
-  stage: tests
-  script:
-    - go get -u github.com/gordonklaus/ineffassign
-    - ineffassign pkg/*.go
-
-misspell:
-  <<: *tests-setup
-  stage: tests
-  script:
-    - go get -u github.com/client9/misspell/cmd/misspell
-    - misspell pkg/*.go
-
-# build-one jobs build packages for a single OS / ARCH combination.
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
 #
-# They are run during the first stage of the pipeline as a smoke test to ensure
-# that we can successfully build packages on all of our architectures for a
-# single OS. They are triggered on any change to an MR. No artifacts are
-# produced as part of build-one jobs.
-.build-one-setup: &build-one-setup
-  <<: *build-setup
-  stage: build-one
-  only:
-    - merge_requests
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include:
+  - .common-ci.yml
 
 # build-all jobs build packages for every OS / ARCH combination we support.
 #
 # They are run under two conditions:
 # 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0)
 # 2) Manually by a reviewer just before merging a MR.
-#
-# Unlike build-one jobs, it takes a long time to build the full suite
-# OS / ARCH combinations, so this is optimized to only run once per MR
-# (assuming it all passes). A full set of artifacts including the packages
-# built for each OS / ARCH are produced as a result of these jobs.
-.build-all-setup: &build-all-setup
-  <<: *build-setup
+.build-all-for-arch:
+  variables:
+    # Setting DIST=docker invokes the docker- release targets
+    DIST: docker
+  extends:
+    - .package-build
   stage: build-all
   timeout: 2h 30m
   rules:
     - if: $CI_COMMIT_TAG
       when: always
-    - if: $CI_MERGE_REQUEST_ID
-      when: always
-
-  variables:
-    ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}"
-    ARTIFACTS_DIR: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}"
-    DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_DIR}"
-
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_DIR}
-
-# The full set of build-one jobs organizes to build
-# ubuntu18.04 in parallel on each of our supported ARCHs.
-build-one-amd64:
-  <<: *build-one-setup
-  script:
-    - make ubuntu18.04-amd64
-
-build-one-ppc64le:
-  <<: *build-one-setup
-  script:
-    - make ubuntu18.04-ppc64le
-
-build-one-arm64:
-  <<: *build-one-setup
-  script:
-    - make ubuntu18.04-arm64
 
 # The full set of build-all jobs organized to
 # have builds for each ARCH run in parallel.
 build-all-amd64:
-  <<: *build-all-setup
-  script:
-    - make docker-amd64
+  extends:
+    - .build-all-for-arch
+    - .arch-amd64
 
 build-all-x86_64:
-  <<: *build-all-setup
-  script:
-    - make docker-x86_64
+  extends:
+    - .build-all-for-arch
+    - .arch-x86_64
 
 build-all-ppc64le:
-  <<: *build-all-setup
-  script:
-    - make docker-ppc64le
+  extends:
+    - .build-all-for-arch
+    - .arch-ppc64le
 
 build-all-arm64:
-  <<: *build-all-setup
-  script:
-    - make docker-arm64
+  extends:
+    - .build-all-for-arch
+    - .arch-arm64
 
 build-all-aarch64:
-  <<: *build-all-setup
-  script:
-    - make docker-aarch64
+  extends:
+    - .build-all-for-arch
+    - .arch-aarch64

.gitmodules

@@ -0,0 +1,9 @@
+[submodule "third_party/libnvidia-container"]
+	path = third_party/libnvidia-container
+	url = https://gitlab.com/nvidia/container-toolkit/libnvidia-container.git
+[submodule "third_party/nvidia-container-runtime"]
+	path = third_party/nvidia-container-runtime
+	url = https://gitlab.com/nvidia/container-toolkit/container-runtime.git
+[submodule "third_party/nvidia-docker"]
+	path = third_party/nvidia-docker
+	url = https://gitlab.com/nvidia/container-toolkit/nvidia-docker.git

.nvidia-ci.yml

@@ -0,0 +1,174 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include:
+  - local: '.common-ci.yml'
+
+default:
+  tags:
+    - cnt
+    - container-dev
+    - docker/multi-arch
+    - docker/privileged
+    - os/linux
+    - type/docker
+
+variables:
+  DOCKER_DRIVER: overlay2
+  DOCKER_TLS_CERTDIR: "/certs"
+  # Release "devel"-tagged images off the master branch
+  RELEASE_DEVEL_BRANCH: "master"
+  DEVEL_RELEASE_IMAGE_VERSION: "devel"
+  # On the multi-arch builder we don't need the qemu setup.
+  SKIP_QEMU_SETUP: "1"
+
+# We skip the integration tests for the internal CI:
+.integration:
+  stage: test
+  before_script:
+    - echo "Skipped in internal CI"
+  script:
+    - echo "Skipped in internal CI"
+
+# The .scan step forms the base of the image scan operation performed before releasing
+# images.
+.scan:
+  stage: scan
+  image: "${PULSE_IMAGE}"
+  variables:
+    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
+    IMAGE_ARCHIVE: "container-toolkit.tar"
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[skip[ _-]scans?\]/i
+      when: never
+    - if: $SKIP_SCANS
+      when: never
+    - if: $CI_COMMIT_TAG == null && $CI_COMMIT_BRANCH != $RELEASE_DEVEL_BRANCH
+      allow_failure: true
+  before_script:
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    # TODO: We should specify the architecture here and scan all architectures
+    - docker pull "${IMAGE}"
+    - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
+    - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
+    - >
+      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
+    - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+  script:
+    - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+  artifacts:
+    when: always
+    expire_in: 1 week
+    paths:
+      - pulse-cli.log
+      - licenses.json
+      - sbom.json
+      - vulns.json
+      - policy_evaluation.json
+
+# Define the scan targets
+scan-centos7:
+  extends:
+    - .scan
+    - .dist-centos7
+  needs:
+    - image-centos7
+
+scan-centos8:
+  extends:
+    - .scan
+    - .dist-centos8
+  needs:
+    - image-centos8
+
+scan-ubuntu18.04:
+  extends:
+    - .scan
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+scan-ubi8:
+  extends:
+    - .scan
+    - .dist-ubi8
+  needs:
+    - image-ubi8
+
+# Define external release helpers
+.release:ngc:
+  extends:
+    - .release:external
+  variables:
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
+    OUT_REGISTRY: "${NGC_REGISTRY}"
+    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    # TODO: For now we disable external releases
+    DOCKER: echo
+
+.release:dockerhub:
+  extends:
+    - .release:external
+  variables:
+    OUT_REGISTRY_USER: "${REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${REGISTRY_TOKEN}"
+    OUT_REGISTRY: "${DOCKERHUB_REGISTRY}"
+    OUT_IMAGE_NAME: "${REGISTRY_IMAGE}"
+
+    # TODO: For now we disable external releases
+    DOCKER: echo
+
+# Define the external release targets
+# Release to NGC
+release:ngc-centos7:
+  extends:
+    - .release:ngc
+    - .dist-centos7
+
+release:ngc-centos8:
+  extends:
+    - .release:ngc
+    - .dist-centos8
+
+release:ngc-ubuntu18:
+  extends:
+    - .release:ngc
+    - .dist-ubuntu18.04
+
+release:ngc-ubi8:
+  extends:
+    - .release:ngc
+    - .dist-ubi8
+
+# Release to Dockerhub
+release:dockerhub-centos7:
+  extends:
+    - .release:dockerhub
+    - .dist-centos7
+
+release:dockerhub-centos8:
+  extends:
+    - .release:dockerhub
+    - .dist-centos8
+
+release:dockerhub-ubuntu18:
+  extends:
+    - .release:dockerhub
+    - .dist-ubuntu18.04
+
+release:dockerhub-ubi8:
+  extends:
+    - .release:dockerhub
+    - .dist-ubi8

DEVELOPMENT.md

@@ -0,0 +1,45 @@
+# NVIDIA Container Toolkit Release Tooling
+
+This repository allows for the components of the NVIDIA container stack to be
+built and released as the NVIDIA Container Toolkit from a single repository. The components:
+* `libnvidia-container`
+* `nvidia-container-runtime`
+* `nvidia-docker`
+are included as submodules in the `third_party` folder.
+
+The `nvidia-container-toolkit` resides in this repo directly.
+
+## Building
+
+In oder to build the packages, the following command is executed
+```sh
+./scripts/build-all-components.sh TARGET
+```
+where `TARGET` is a make target that is valid for each of the sub-components.
+
+These include:
+* `ubuntu18.04-amd64`
+* `centos8-x86_64`
+
+The packages are generated in the `dist` folder.
+
+## Testing local changes
+
+In oder to use the same build logic to be used to generate packages with local changes,
+the location of the individual components can be overridded using the: `LIBNVIDIA_CONTAINER_ROOT`,
+`NVIDIA_CONTAINER_TOOLKIT_ROOT`, `NVIDIA_CONTAINER_RUNTIME_ROOT`, and `NVIDIA_DOCKER_ROOT`
+environment variables.
+
+## Testing packages locally
+
+The [test/release](./test/release/) folder contains documentation on how the installation of local or staged packages can be tested.
+
+
+## Releasing
+
+A utility script [`scripts/release.sh`](./scripts/release.sh) is provided to build
+packages required for release. If run without arguments, all supported distribution-architecture combinations are built. A specific distribution-architecture pair can also be provided
+```sh
+./scripts/release.sh ubuntu18.04-amd64
+```
+where the `amd64` builds for `ubuntu18.04` are provided as an example.

Jenkinsfile

@@ -0,0 +1,142 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+podTemplate (cloud:'sw-gpu-cloudnative',
+    containers: [
+    containerTemplate(name: 'docker', image: 'docker:dind', ttyEnabled: true, privileged: true),
+    containerTemplate(name: 'golang', image: 'golang:1.16.3', ttyEnabled: true)
+  ]) {
+    node(POD_LABEL) {
+        def scmInfo
+
+        stage('checkout') {
+            scmInfo = checkout(scm)
+        }
+
+        stage('dependencies') {
+            container('golang') {
+                sh 'GO111MODULE=off go get -u github.com/client9/misspell/cmd/misspell'
+                sh 'GO111MODULE=off go get -u github.com/gordonklaus/ineffassign'
+                sh 'GO111MODULE=off go get -u golang.org/x/lint/golint'
+            }
+            container('docker') {
+                sh 'apk add --no-cache make bash git'
+            }
+        }
+        stage('check') {
+            parallel (
+                getGolangStages(["assert-fmt", "lint", "vet", "ineffassign", "misspell"])
+            )
+        }
+        stage('test') {
+            parallel (
+                getGolangStages(["test"])
+            )
+        }
+
+        def versionInfo
+        stage('version') {
+            container('docker') {
+                versionInfo = getVersionInfo(scmInfo)
+                println "versionInfo=${versionInfo}"
+            }
+        }
+
+        def dist = 'ubuntu20.04'
+        def arch = 'amd64'
+        def stageLabel = "${dist}-${arch}"
+
+        stage('build-one') {
+            container('docker') {
+                stage (stageLabel) {
+                    sh "make ${dist}-${arch}"
+                }
+            }
+        }
+
+        stage('release') {
+            container('docker') {
+                stage (stageLabel) {
+
+                    def component = 'main'
+                    def repository = 'sw-gpu-cloudnative-debian-local/pool/main/'
+
+                    def uploadSpec = """{
+                                        "files":
+                                        [  {
+                                                "pattern": "./dist/${dist}/${arch}/*.deb",
+                                                "target": "${repository}",
+                                                "props": "deb.distribution=${dist};deb.component=${component};deb.architecture=${arch}"
+                                            }
+                                        ]
+                                    }"""
+
+                    sh "echo starting release with versionInfo=${versionInfo}"
+                    if (versionInfo.isTag) {
+                        // upload to artifactory repository
+                        def server = Artifactory.server 'sw-gpu-artifactory'
+                        server.upload spec: uploadSpec
+                    } else {
+                        sh "echo skipping release for non-tagged build"
+                    }
+                }
+            }
+        }
+    }
+}
+
+def getGolangStages(def targets) {
+    stages = [:]
+
+    for (t in targets) {
+        stages[t] = getLintClosure(t)
+    }
+
+    return stages
+}
+
+def getLintClosure(def target) {
+    return {
+        container('golang') {
+            stage(target) {
+                sh "make ${target}"
+            }
+        }
+    }
+}
+
+// getVersionInfo returns a hash of version info
+def getVersionInfo(def scmInfo) {
+    def versionInfo = [
+        isTag: isTag(scmInfo.GIT_BRANCH)
+    ]
+
+    scmInfo.each { k, v -> versionInfo[k] = v }
+    return versionInfo
+}
+
+def isTag(def branch) {
+    if (!branch.startsWith('v')) {
+        return false
+    }
+
+    def version = shOutput('git describe --all --exact-match --always')
+    return version == "tags/${branch}"
+}
+
+def shOuptut(def script) {
+    return sh(script: script, returnStdout: true).trim()
+}

Makefile

@@ -1,19 +1,147 @@
-# Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 DOCKER   ?= docker
 MKDIR    ?= mkdir
 DIST_DIR ?= $(CURDIR)/dist
 
 LIB_NAME := nvidia-container-toolkit
-LIB_VERSION := 1.3.0
-LIB_TAG ?=
+LIB_VERSION := 1.7.0
+LIB_TAG := rc.1
 
-GOLANG_VERSION := 1.14.2
-GOLANG_PKG_PATH := github.com/NVIDIA/nvidia-container-toolkit/pkg
+GOLANG_VERSION := 1.16.3
+MODULE := github.com/NVIDIA/nvidia-container-toolkit
 
 # By default run all native docker-based targets
 docker-native:
-include $(CURDIR)/docker.mk
+include $(CURDIR)/docker/docker.mk
 
-binary:
-	go build -ldflags "-s -w" -o "$(LIB_NAME)" $(GOLANG_PKG_PATH)
+ifeq ($(IMAGE_NAME),)
+REGISTRY ?= nvidia
+IMAGE_NAME = $(REGISTRY)/container-toolkit
+endif
+
+BUILDIMAGE_TAG ?= golang$(GOLANG_VERSION)
+BUILDIMAGE ?= $(IMAGE_NAME)-build:$(BUILDIMAGE_TAG)
+
+EXAMPLES := $(patsubst ./examples/%/,%,$(sort $(dir $(wildcard ./examples/*/))))
+EXAMPLE_TARGETS := $(patsubst %,example-%, $(EXAMPLES))
+
+CMDS := $(patsubst ./cmd/%/,%,$(sort $(dir $(wildcard ./cmd/*/))))
+CMD_TARGETS := $(patsubst %,cmd-%, $(CMDS))
+
+$(info CMD_TARGETS=$(CMD_TARGETS))
+
+CHECK_TARGETS := assert-fmt vet lint ineffassign misspell
+MAKE_TARGETS := binaries build check fmt lint-internal test examples cmds coverage generate $(CHECK_TARGETS)
+
+TARGETS := $(MAKE_TARGETS) $(EXAMPLE_TARGETS) $(CMD_TARGETS)
+
+DOCKER_TARGETS := $(patsubst %,docker-%, $(TARGETS))
+.PHONY: $(TARGETS) $(DOCKER_TARGETS)
+
+GOOS ?= linux
+
+binaries: cmds
+ifneq ($(PREFIX),)
+cmd-%: COMMAND_BUILD_OPTIONS = -o $(PREFIX)/$(*)
+endif
+cmds: $(CMD_TARGETS)
+$(CMD_TARGETS): cmd-%:
+	GOOS=$(GOOS) go build -ldflags "-s -w" $(COMMAND_BUILD_OPTIONS) $(MODULE)/cmd/$(*)
+
+build:
+	GOOS=$(GOOS) go build ./...
+
+examples: $(EXAMPLE_TARGETS)
+$(EXAMPLE_TARGETS): example-%:
+	GOOS=$(GOOS) go build ./examples/$(*)
+
+all: check test build binary
+check: $(CHECK_TARGETS)
+
+# Apply go fmt to the codebase
+fmt:
+	go list -f '{{.Dir}}' $(MODULE)/... \
+		| xargs gofmt -s -l -w
+
+assert-fmt:
+	go list -f '{{.Dir}}' $(MODULE)/... \
+		| xargs gofmt -s -l > fmt.out
+	@if [ -s fmt.out ]; then \
+		echo "\nERROR: The following files are not formatted:\n"; \
+		cat fmt.out; \
+		rm fmt.out; \
+		exit 1; \
+	else \
+		rm fmt.out; \
+	fi
+
+ineffassign:
+	ineffassign $(MODULE)/...
+
+lint:
+# We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
+	go list -f '{{.Dir}}' $(MODULE)/... | grep -v /internal/ | xargs golint -set_exit_status
+
+lint-internal:
+# We use `go list -f '{{.Dir}}' $(MODULE)/...` to skip the `vendor` folder.
+	go list -f '{{.Dir}}' $(MODULE)/internal/... | xargs golint -set_exit_status
+
+misspell:
+	misspell $(MODULE)/...
+
+vet:
+	go vet $(MODULE)/...
+
+COVERAGE_FILE := coverage.out
+test: build cmds
+	go test -v -coverprofile=$(COVERAGE_FILE) $(MODULE)/...
+
+coverage: test
+	cat $(COVERAGE_FILE) | grep -v "_mock.go" > $(COVERAGE_FILE).no-mocks
+	go tool cover -func=$(COVERAGE_FILE).no-mocks
+
+generate:
+	go generate $(MODULE)/...
+
+# Generate an image for containerized builds
+# Note: This image is local only
+.PHONY: .build-image .pull-build-image .push-build-image
+.build-image: docker/Dockerfile.devel
+	if [ x"$(SKIP_IMAGE_BUILD)" = x"" ]; then \
+		$(DOCKER) build \
+			--progress=plain \
+			--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
+			--tag $(BUILDIMAGE) \
+			-f $(^) \
+			docker; \
+	fi
+
+.pull-build-image:
+	$(DOCKER) pull $(BUILDIMAGE)
+
+.push-build-image:
+	$(DOCKER) push $(BUILDIMAGE)
+
+$(DOCKER_TARGETS): docker-%: .build-image
+	@echo "Running 'make $(*)' in docker container $(BUILDIMAGE)"
+	$(DOCKER) run \
+		--rm \
+		-e GOCACHE=/tmp/.cache \
+		-v $(PWD):$(PWD) \
+		-w $(PWD) \
+		--user $$(id -u):$$(id -g) \
+		$(BUILDIMAGE) \
+			make $(*)

README.md

@@ -0,0 +1,31 @@
+# NVIDIA Container Toolkit
+
+[![GitHub license](https://img.shields.io/github/license/NVIDIA/nvidia-container-toolkit?style=flat-square)](https://raw.githubusercontent.com/NVIDIA/nvidia-container-toolkit/master/LICENSE)
+[![Documentation](https://img.shields.io/badge/documentation-wiki-blue.svg?style=flat-square)](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/overview.html)
+[![Package repository](https://img.shields.io/badge/packages-repository-b956e8.svg?style=flat-square)](https://nvidia.github.io/libnvidia-container)
+
+![nvidia-container-stack](https://cloud.githubusercontent.com/assets/3028125/12213714/5b208976-b632-11e5-8406-38d379ec46aa.png)
+
+## Introduction
+
+The NVIDIA Container Toolkit allows users to build and run GPU accelerated containers. The toolkit includes a container runtime [library](https://github.com/NVIDIA/libnvidia-container) and utilities to automatically configure containers to leverage NVIDIA GPUs.
+
+Product documentation including an architecture overview, platform support, and installation and usage guides can be found in the [documentation repository](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/overview.html).
+
+## Getting Started
+
+**Make sure you have installed the [NVIDIA driver](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#nvidia-drivers) for your Linux Distribution**
+**Note that you do not need to install the CUDA Toolkit on the host system, but the NVIDIA driver needs to be installed**
+
+For instructions on getting started with the NVIDIA Container Toolkit, refer to the [installation guide](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#installation-guide).
+
+## Usage
+
+The [user guide](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/user-guide.html) provides information on the configuration and command line options available when running GPU containers with Docker.
+
+## Issues and Contributing
+
+[Checkout the Contributing document!](CONTRIBUTING.md)
+
+* Please let us know by [filing a new issue](https://github.com/NVIDIA/nvidia-container-toolkit/issues/new)
+* You can contribute by creating a [merge request](https://gitlab.com/nvidia/container-toolkit/container-toolkit/-/merge_requests/new) to our public GitLab repository

build/container/Dockerfile.centos

@@ -0,0 +1,76 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+# NOTE: In cases where the libc version is a concern, we would have to use an
+# image based on the target OS to build the golang executables here -- especially
+# if cgo code is included.
+FROM golang:${GOLANG_VERSION} as build
+
+# We override the GOPATH to ensure that the binaries are installed to
+# /artifacts/bin
+ARG GOPATH=/artifacts
+
+# Install the experiemental nvidia-container-runtime
+# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
+ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+
+WORKDIR /build
+COPY . .
+
+# NOTE: Until the config utilities are properly integrated into the
+# nvidia-container-toolkit repository, these are built from the `tools` folder
+# and not `cmd`.
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+WORKDIR /artifacts/packages
+
+ARG ARTIFACTS_DIR
+COPY ${ARTIFACTS_DIR}/* /artifacts/packages/
+
+ARG PACKAGE_VERSION
+RUN yum localinstall -y \
+    libnvidia-container1-${PACKAGE_VERSION}*.rpm \
+    libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
+    nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
+
+WORKDIR /work
+
+COPY --from=build /artifacts/bin /work
+
+ENV PATH=/work:$PATH
+
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+COPY ./LICENSE /licenses/LICENSE
+
+ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.ubuntu

@@ -0,0 +1,82 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+# NOTE: In cases where the libc version is a concern, we would have to use an
+# image based on the target OS to build the golang executables here -- especially
+# if cgo code is included.
+FROM golang:${GOLANG_VERSION} as build
+
+# We override the GOPATH to ensure that the binaries are installed to
+# /artifacts/bin
+ARG GOPATH=/artifacts
+
+# Install the experiemental nvidia-container-runtime
+# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
+ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+
+WORKDIR /build
+COPY . .
+
+# NOTE: Until the config utilities are properly integrated into the
+# nvidia-container-toolkit repository, these are built from the `tools` folder
+# and not `cmd`.
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libcap2 \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+WORKDIR /artifacts/packages
+
+ARG ARTIFACTS_DIR
+COPY ${ARTIFACTS_DIR}/* /artifacts/packages/
+
+ARG PACKAGE_VERSION
+RUN dpkg -i \
+    libnvidia-container1_${PACKAGE_VERSION}*.deb \
+    libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
+    nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
+
+WORKDIR /work
+
+COPY --from=build /artifacts/bin /work/
+
+ENV PATH=/work:$PATH
+
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+COPY ./LICENSE /licenses/LICENSE
+
+ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -0,0 +1,110 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DOCKER   ?= docker
+MKDIR    ?= mkdir
+DIST_DIR ?= $(CURDIR)/dist
+
+##### Global variables #####
+
+# TODO: These should be defined ONCE and currently duplicate the version in the
+# toolkit makefile.
+LIB_VERSION := 1.7.0
+LIB_TAG := rc.1
+
+VERSION ?= $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+
+CUDA_VERSION ?= 11.4.2
+GOLANG_VERSION ?= 1.16.4
+ifeq ($(IMAGE_NAME),)
+REGISTRY ?= nvidia
+IMAGE_NAME := $(REGISTRY)/container-toolkit
+endif
+
+IMAGE_TAG ?= $(VERSION)-$(DIST)
+IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
+
+##### Public rules #####
+DEFAULT_PUSH_TARGET := ubuntu18.04
+TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
+
+BUILD_TARGETS := $(patsubst %, build-%, $(TARGETS))
+PUSH_TARGETS := $(patsubst %, push-%, $(TARGETS))
+TEST_TARGETS := $(patsubst %, test-%, $(TARGETS))
+
+.PHONY: $(TARGETS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
+
+$(PUSH_TARGETS): push-%:
+	$(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)"
+
+# For the default push target we also push a short tag equal to the version.
+# We skip this for the development release
+DEVEL_RELEASE_IMAGE_VERSION ?= devel
+ifneq ($(strip $(VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
+push-$(DEFAULT_PUSH_TARGET): push-short
+endif
+push-short:
+	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(IMAGE_NAME):$(VERSION)"
+	$(DOCKER) push "$(IMAGE_NAME):$(VERSION)"
+
+
+build-%: DIST = $(*)
+build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
+
+# Use a generic build target to build the relevant images
+$(BUILD_TARGETS): build-%: $(ARTIFACTS_DIR)
+	$(DOCKER) build --pull \
+		--tag $(IMAGE) \
+		--build-arg ARTIFACTS_DIR="$(ARTIFACTS_DIR)" \
+		--build-arg BASE_DIST="$(BASE_DIST)" \
+		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
+		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
+		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
+		--build-arg VERSION="$(VERSION)" \
+		-f $(DOCKERFILE) \
+		$(CURDIR)
+
+
+ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
+
+build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
+build-ubuntu%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/amd64
+build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+
+build-ubuntu18.04: BASE_DIST := ubuntu18.04
+build-ubuntu20.04: BASE_DIST := ubuntu20.04
+
+build-ubi8: DOCKERFILE_SUFFIX := centos
+# TODO: Update this to use the centos8 packages
+build-ubi8: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/centos7/x86_64
+build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-ubi8: BASE_DIST := ubi8
+
+build-centos%: DOCKERFILE_SUFFIX := centos
+build-centos%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/x86_64
+build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+
+build-centos7: BASE_DIST := centos7
+build-centos8: BASE_DIST := centos8
+
+# Test targets
+test-%: DIST = $(*)
+
+TEST_CASES ?= toolkit docker crio containerd
+$(TEST_TARGETS): test-%:
+	TEST_CASES="$(TEST_CASES)" bash -x $(CURDIR)/test/container/main.sh run \
+		$(CURDIR)/shared-$(*) \
+		$(IMAGE) \
+			--no-cleanup-on-error
+

build/container/README.md

@@ -0,0 +1,4 @@
+# NVIDIA Container Toolkit Container
+
+This folder contains make and docker files for building the NVIDIA Container Toolkit Container.
+

cmd/nvidia-container-runtime/logger.go

@@ -0,0 +1,79 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/tsaikd/KDGoLib/logrusutil"
+)
+
+// Logger adds a way to manage output to a log file to a logrus.Logger
+type Logger struct {
+	*logrus.Logger
+	previousOutput io.Writer
+	logFile        *os.File
+}
+
+// NewLogger constructs a Logger with a preddefined formatter
+func NewLogger() *Logger {
+	logrusLogger := logrus.New()
+
+	formatter := &logrusutil.ConsoleLogFormatter{
+		TimestampFormat: "2006/01/02 15:04:07",
+		Flag:            logrusutil.Ltime,
+	}
+
+	logger := &Logger{
+		Logger: logrusLogger,
+	}
+	logger.SetFormatter(formatter)
+
+	return logger
+}
+
+// LogToFile opens the specified file for appending and sets the logger to
+// output to the opened file. A reference to the file pointer is stored to
+// allow this to be closed.
+func (l *Logger) LogToFile(filename string) error {
+	logFile, err := os.OpenFile(filename, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("error opening debug log file: %v", err)
+	}
+
+	l.logFile = logFile
+	l.previousOutput = l.Out
+	l.SetOutput(logFile)
+
+	return nil
+}
+
+// CloseFile closes the log file (if any) and resets the logger output to what it
+// was before LogToFile was called.
+func (l *Logger) CloseFile() error {
+	if l.logFile == nil {
+		return nil
+	}
+	logFile := l.logFile
+	l.SetOutput(l.previousOutput)
+	l.logFile = nil
+
+	return logFile.Close()
+}

cmd/nvidia-container-runtime/main.go

@@ -0,0 +1,89 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"path"
+
+	"github.com/pelletier/go-toml"
+)
+
+const (
+	configOverride = "XDG_CONFIG_HOME"
+	configFilePath = "nvidia-container-runtime/config.toml"
+
+	hookDefaultFilePath = "/usr/bin/nvidia-container-runtime-hook"
+)
+
+var (
+	configDir = "/etc/"
+)
+
+var logger = NewLogger()
+
+func main() {
+	err := run(os.Args)
+	if err != nil {
+		logger.Errorf("Error running %v: %v", os.Args, err)
+		os.Exit(1)
+	}
+}
+
+// run is an entry point that allows for idiomatic handling of errors
+// when calling from the main function.
+func run(argv []string) (err error) {
+	cfg, err := getConfig()
+	if err != nil {
+		return fmt.Errorf("error loading config: %v", err)
+	}
+
+	err = logger.LogToFile(cfg.debugFilePath)
+	if err != nil {
+		return fmt.Errorf("error opening debug log file: %v", err)
+	}
+	defer func() {
+		// We capture and log a returning error before closing the log file.
+		if err != nil {
+			logger.Errorf("Error running %v: %v", argv, err)
+		}
+		logger.CloseFile()
+	}()
+
+	r, err := newRuntime(argv)
+	if err != nil {
+		return fmt.Errorf("error creating runtime: %v", err)
+	}
+
+	logger.Printf("Running %s\n", argv[0])
+	return r.Exec(argv)
+}
+
+type config struct {
+	debugFilePath string
+}
+
+// getConfig sets up the config struct. Values are read from a toml file
+// or set via the environment.
+func getConfig() (*config, error) {
+	cfg := &config{}
+
+	if XDGConfigDir := os.Getenv(configOverride); len(XDGConfigDir) != 0 {
+		configDir = XDGConfigDir
+	}
+
+	configFilePath := path.Join(configDir, configFilePath)
+
+	tomlContent, err := os.ReadFile(configFilePath)
+	if err != nil {
+		return nil, err
+	}
+
+	toml, err := toml.Load(string(tomlContent))
+	if err != nil {
+		return nil, err
+	}
+
+	cfg.debugFilePath = toml.GetDefault("nvidia-container-runtime.debug", "/dev/null").(string)
+
+	return cfg, nil
+}

cmd/nvidia-container-runtime/main_test.go

@@ -0,0 +1,293 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	nvidiaRuntime            = "nvidia-container-runtime"
+	nvidiaHook               = "nvidia-container-runtime-hook"
+	bundlePathSuffix         = "test/output/bundle/"
+	specFile                 = "config.json"
+	unmodifiedSpecFileSuffix = "test/input/test_spec.json"
+)
+
+type testConfig struct {
+	root    string
+	binPath string
+}
+
+var cfg *testConfig
+
+func TestMain(m *testing.M) {
+	// TEST SETUP
+	// Determine the module root and the test binary path
+	var err error
+	moduleRoot, err := getModuleRoot()
+	if err != nil {
+		logger.Fatalf("error in test setup: could not get module root: %v", err)
+	}
+	testBinPath := filepath.Join(moduleRoot, "test", "bin")
+	testInputPath := filepath.Join(moduleRoot, "test", "input")
+
+	// Set the environment variables for the test
+	os.Setenv("PATH", prependToPath(testBinPath, moduleRoot))
+	os.Setenv("XDG_CONFIG_HOME", testInputPath)
+
+	// Confirm that the environment is configured correctly
+	runcPath, err := exec.LookPath(runcExecutableName)
+	if err != nil || filepath.Join(testBinPath, runcExecutableName) != runcPath {
+		logger.Fatalf("error in test setup: mock runc path set incorrectly in TestMain(): %v", err)
+	}
+	hookPath, err := exec.LookPath(nvidiaHook)
+	if err != nil || filepath.Join(testBinPath, nvidiaHook) != hookPath {
+		logger.Fatalf("error in test setup: mock hook path set incorrectly in TestMain(): %v", err)
+	}
+
+	// Store the root and binary paths in the test Config
+	cfg = &testConfig{
+		root:    moduleRoot,
+		binPath: testBinPath,
+	}
+
+	// RUN TESTS
+	exitCode := m.Run()
+
+	// TEST CLEANUP
+	os.Remove(specFile)
+
+	os.Exit(exitCode)
+}
+
+func getModuleRoot() (string, error) {
+	_, filename, _, _ := runtime.Caller(0)
+
+	return hasGoMod(filename)
+}
+
+func hasGoMod(dir string) (string, error) {
+	if dir == "" || dir == "/" {
+		return "", fmt.Errorf("module root not found")
+	}
+
+	_, err := os.Stat(filepath.Join(dir, "go.mod"))
+	if err != nil {
+		return hasGoMod(filepath.Dir(dir))
+	}
+	return dir, nil
+}
+
+func prependToPath(additionalPaths ...string) string {
+	paths := strings.Split(os.Getenv("PATH"), ":")
+	paths = append(additionalPaths, paths...)
+
+	return strings.Join(paths, ":")
+}
+
+// case 1) nvidia-container-runtime run --bundle
+// case 2) nvidia-container-runtime create --bundle
+//		- Confirm the runtime handles bad input correctly
+func TestBadInput(t *testing.T) {
+	err := cfg.generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle")
+	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
+	output, err := cmdRun.CombinedOutput()
+	require.Errorf(t, err, "runtime should return an error", "output=%v", string(output))
+
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	err = cmdCreate.Run()
+	require.Error(t, err, "runtime should return an error")
+}
+
+// case 1) nvidia-container-runtime run --bundle <bundle-name> <ctr-name>
+//		- Confirm the runtime runs with no errors
+// case 2) nvidia-container-runtime create --bundle <bundle-name> <ctr-name>
+//		- Confirm the runtime inserts the NVIDIA prestart hook correctly
+func TestGoodInput(t *testing.T) {
+	err := cfg.generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatalf("error generating runtime spec: %v", err)
+	}
+
+	cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle", cfg.bundlePath(), "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " "))
+	output, err := cmdRun.CombinedOutput()
+	require.NoErrorf(t, err, "runtime should not return an error", "output=%v", string(output))
+
+	// Check config.json and confirm there are no hooks
+	spec, err := cfg.getRuntimeSpec()
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.Empty(t, spec.Hooks, "there should be no hooks in config.json")
+
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	err = cmdCreate.Run()
+	require.NoError(t, err, "runtime should not return an error")
+
+	// Check config.json for NVIDIA prestart hook
+	spec, err = cfg.getRuntimeSpec()
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.NotEmpty(t, spec.Hooks, "there should be hooks in config.json")
+	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
+}
+
+// NVIDIA prestart hook already present in config file
+func TestDuplicateHook(t *testing.T) {
+	err := cfg.generateNewRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var spec specs.Spec
+	spec, err = cfg.getRuntimeSpec()
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	t.Logf("inserting nvidia prestart hook to config.json")
+	if err = addNVIDIAHook(&spec); err != nil {
+		t.Fatal(err)
+	}
+
+	jsonOutput, err := json.MarshalIndent(spec, "", "\t")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	jsonFile, err := os.OpenFile(cfg.specFilePath(), os.O_RDWR, 0644)
+	if err != nil {
+		t.Fatal(err)
+	}
+	_, err = jsonFile.WriteAt(jsonOutput, 0)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	// Test how runtime handles already existing prestart hook in config.json
+	cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle", cfg.bundlePath(), "testcontainer")
+	t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " "))
+	output, err := cmdCreate.CombinedOutput()
+	require.NoErrorf(t, err, "runtime should not return an error", "output=%v", string(output))
+
+	// Check config.json for NVIDIA prestart hook
+	spec, err = cfg.getRuntimeSpec()
+	require.NoError(t, err, "should be no errors when reading and parsing spec from config.json")
+	require.NotEmpty(t, spec.Hooks, "there should be hooks in config.json")
+	require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "exactly one nvidia prestart hook should be inserted correctly into config.json")
+}
+
+// addNVIDIAHook is a basic wrapper for nvidiaContainerRunime.addNVIDIAHook that is used for
+// testing.
+func addNVIDIAHook(spec *specs.Spec) error {
+	r := nvidiaContainerRuntime{logger: logger.Logger}
+	return r.addNVIDIAHook(spec)
+}
+
+func (c testConfig) getRuntimeSpec() (specs.Spec, error) {
+	filePath := c.specFilePath()
+
+	var spec specs.Spec
+	jsonFile, err := os.OpenFile(filePath, os.O_RDWR, 0644)
+	if err != nil {
+		return spec, err
+	}
+	defer jsonFile.Close()
+
+	jsonContent, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		return spec, err
+	} else if json.Valid(jsonContent) {
+		err = json.Unmarshal(jsonContent, &spec)
+		if err != nil {
+			return spec, err
+		}
+	} else {
+		err = json.NewDecoder(bytes.NewReader(jsonContent)).Decode(&spec)
+		if err != nil {
+			return spec, err
+		}
+	}
+
+	return spec, err
+}
+
+func (c testConfig) bundlePath() string {
+	return filepath.Join(c.root, bundlePathSuffix)
+}
+
+func (c testConfig) specFilePath() string {
+	return filepath.Join(c.bundlePath(), specFile)
+}
+
+func (c testConfig) unmodifiedSpecFile() string {
+	return filepath.Join(c.root, unmodifiedSpecFileSuffix)
+}
+
+func (c testConfig) generateNewRuntimeSpec() error {
+	var err error
+
+	err = os.MkdirAll(c.bundlePath(), 0755)
+	if err != nil {
+		return err
+	}
+
+	cmd := exec.Command("cp", c.unmodifiedSpecFile(), c.specFilePath())
+	err = cmd.Run()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+// Return number of valid NVIDIA prestart hooks in runtime spec
+func nvidiaHookCount(hooks *specs.Hooks) int {
+	if hooks == nil {
+		return 0
+	}
+
+	count := 0
+	for _, hook := range hooks.Prestart {
+		if strings.Contains(hook.Path, nvidiaHook) {
+			count++
+		}
+	}
+	return count
+}
+
+func TestGetConfigWithCustomConfig(t *testing.T) {
+	wd, err := os.Getwd()
+	require.NoError(t, err)
+
+	// By default debug is disabled
+	contents := []byte("[nvidia-container-runtime]\ndebug = \"/nvidia-container-toolkit.log\"")
+	testDir := filepath.Join(wd, "test")
+	filename := filepath.Join(testDir, configFilePath)
+
+	os.Setenv(configOverride, testDir)
+
+	require.NoError(t, os.MkdirAll(filepath.Dir(filename), 0766))
+	require.NoError(t, ioutil.WriteFile(filename, contents, 0766))
+
+	defer func() { require.NoError(t, os.RemoveAll(testDir)) }()
+
+	cfg, err := getConfig()
+	require.NoError(t, err)
+	require.Equal(t, cfg.debugFilePath, "/nvidia-container-toolkit.log")
+}

cmd/nvidia-container-runtime/nvcr.go

@@ -0,0 +1,132 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	log "github.com/sirupsen/logrus"
+)
+
+// nvidiaContainerRuntime encapsulates the NVIDIA Container Runtime. It wraps the specified runtime, conditionally
+// modifying the specified OCI specification before invoking the runtime.
+type nvidiaContainerRuntime struct {
+	logger  *log.Logger
+	runtime oci.Runtime
+	ociSpec oci.Spec
+}
+
+var _ oci.Runtime = (*nvidiaContainerRuntime)(nil)
+
+// newNvidiaContainerRuntime is a constructor for a standard runtime shim.
+func newNvidiaContainerRuntimeWithLogger(logger *log.Logger, runtime oci.Runtime, ociSpec oci.Spec) (oci.Runtime, error) {
+	r := nvidiaContainerRuntime{
+		logger:  logger,
+		runtime: runtime,
+		ociSpec: ociSpec,
+	}
+
+	return &r, nil
+}
+
+// Exec defines the entrypoint for the NVIDIA Container Runtime. A check is performed to see whether modifications
+// to the OCI spec are required -- and applicable modifcations applied. The supplied arguments are then
+// forwarded to the underlying runtime's Exec method.
+func (r nvidiaContainerRuntime) Exec(args []string) error {
+	if r.modificationRequired(args) {
+		err := r.modifyOCISpec()
+		if err != nil {
+			return fmt.Errorf("error modifying OCI spec: %v", err)
+		}
+	}
+
+	r.logger.Println("Forwarding command to runtime")
+	return r.runtime.Exec(args)
+}
+
+// modificationRequired checks the intput arguments to determine whether a modification
+// to the OCI spec is required.
+func (r nvidiaContainerRuntime) modificationRequired(args []string) bool {
+	if oci.HasCreateSubcommand(args) {
+		r.logger.Infof("'create' command detected; modification required")
+		return true
+	}
+
+	r.logger.Infof("No modification required")
+	return false
+}
+
+// modifyOCISpec loads and modifies the OCI spec specified in the nvidiaContainerRuntime
+// struct. The spec is modified in-place and written to the same file as the input after
+// modifcationas are applied.
+func (r nvidiaContainerRuntime) modifyOCISpec() error {
+	err := r.ociSpec.Load()
+	if err != nil {
+		return fmt.Errorf("error loading OCI specification for modification: %v", err)
+	}
+
+	err = r.ociSpec.Modify(r.addNVIDIAHook)
+	if err != nil {
+		return fmt.Errorf("error injecting NVIDIA Container Runtime hook: %v", err)
+	}
+
+	err = r.ociSpec.Flush()
+	if err != nil {
+		return fmt.Errorf("error writing modified OCI specification: %v", err)
+	}
+	return nil
+}
+
+// addNVIDIAHook modifies the specified OCI specification in-place, inserting a
+// prestart hook.
+func (r nvidiaContainerRuntime) addNVIDIAHook(spec *specs.Spec) error {
+	path, err := exec.LookPath("nvidia-container-runtime-hook")
+	if err != nil {
+		path = hookDefaultFilePath
+		_, err = os.Stat(path)
+		if err != nil {
+			return err
+		}
+	}
+
+	r.logger.Printf("prestart hook path: %s\n", path)
+
+	args := []string{path}
+	if spec.Hooks == nil {
+		spec.Hooks = &specs.Hooks{}
+	} else if len(spec.Hooks.Prestart) != 0 {
+		for _, hook := range spec.Hooks.Prestart {
+			if !strings.Contains(hook.Path, "nvidia-container-runtime-hook") {
+				continue
+			}
+			r.logger.Println("existing nvidia prestart hook in OCI spec file")
+			return nil
+		}
+	}
+
+	spec.Hooks.Prestart = append(spec.Hooks.Prestart, specs.Hook{
+		Path: path,
+		Args: append(args, "prestart"),
+	})
+
+	return nil
+}

cmd/nvidia-container-runtime/nvcr_test.go

@@ -0,0 +1,203 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"strings"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestAddNvidiaHook(t *testing.T) {
+	logger, logHook := testlog.NewNullLogger()
+	shim := nvidiaContainerRuntime{
+		logger: logger,
+	}
+
+	testCases := []struct {
+		spec         *specs.Spec
+		errorPrefix  string
+		shouldNotAdd bool
+	}{
+		{
+			spec: &specs.Spec{},
+		},
+		{
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{},
+			},
+		},
+		{
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{{
+						Path: "some-hook",
+					}},
+				},
+			},
+		},
+		{
+			spec: &specs.Spec{
+				Hooks: &specs.Hooks{
+					Prestart: []specs.Hook{{
+						Path: "nvidia-container-runtime-hook",
+					}},
+				},
+			},
+			shouldNotAdd: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		logHook.Reset()
+
+		var numPrestartHooks int
+		if tc.spec.Hooks != nil {
+			numPrestartHooks = len(tc.spec.Hooks.Prestart)
+		}
+
+		err := shim.addNVIDIAHook(tc.spec)
+
+		if tc.errorPrefix == "" {
+			require.NoErrorf(t, err, "%d: %v", i, tc)
+		} else {
+			require.Truef(t, strings.HasPrefix(err.Error(), tc.errorPrefix), "%d: %v", i, tc)
+
+			require.NotNilf(t, tc.spec.Hooks, "%d: %v", i, tc)
+			require.Equalf(t, 1, nvidiaHookCount(tc.spec.Hooks), "%d: %v", i, tc)
+
+			if tc.shouldNotAdd {
+				require.Equal(t, numPrestartHooks+1, len(tc.spec.Hooks.Poststart), "%d: %v", i, tc)
+			} else {
+				require.Equal(t, numPrestartHooks+1, len(tc.spec.Hooks.Poststart), "%d: %v", i, tc)
+
+				nvidiaHook := tc.spec.Hooks.Poststart[len(tc.spec.Hooks.Poststart)-1]
+
+				// TODO: This assumes that the hook has been set up in the makefile
+				expectedPath := "/usr/bin/nvidia-container-runtime-hook"
+				require.Equalf(t, expectedPath, nvidiaHook.Path, "%d: %v", i, tc)
+				require.Equalf(t, []string{expectedPath, "prestart"}, nvidiaHook.Args, "%d: %v", i, tc)
+				require.Emptyf(t, nvidiaHook.Env, "%d: %v", i, tc)
+				require.Nilf(t, nvidiaHook.Timeout, "%d: %v", i, tc)
+			}
+		}
+	}
+}
+
+func TestNvidiaContainerRuntime(t *testing.T) {
+	logger, hook := testlog.NewNullLogger()
+
+	testCases := []struct {
+		shim         nvidiaContainerRuntime
+		shouldModify bool
+		args         []string
+		modifyError  error
+		writeError   error
+	}{
+		{
+			shim:         nvidiaContainerRuntime{},
+			shouldModify: false,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"--bundle=create"},
+			shouldModify: false,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"--bundle", "create"},
+			shouldModify: false,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"create"},
+			modifyError:  fmt.Errorf("error modifying"),
+			shouldModify: true,
+		},
+		{
+			shim:         nvidiaContainerRuntime{},
+			args:         []string{"create"},
+			writeError:   fmt.Errorf("error writing"),
+			shouldModify: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		tc.shim.logger = logger
+		hook.Reset()
+
+		ociMock := &oci.SpecMock{
+			ModifyFunc: func(specModifier oci.SpecModifier) error {
+				return tc.modifyError
+			},
+			FlushFunc: func() error {
+				return tc.writeError
+			},
+		}
+		require.Equal(t, tc.shouldModify, tc.shim.modificationRequired(tc.args), "%d: %v", i, tc)
+
+		tc.shim.ociSpec = ociMock
+		tc.shim.runtime = &MockShim{}
+
+		err := tc.shim.Exec(tc.args)
+		if tc.modifyError != nil || tc.writeError != nil {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+
+		if tc.shouldModify {
+			require.Equal(t, 1, len(ociMock.ModifyCalls()), "%d: %v", i, tc)
+		} else {
+			require.Equal(t, 0, len(ociMock.ModifyCalls()), "%d: %v", i, tc)
+		}
+
+		writeExpected := tc.shouldModify && tc.modifyError == nil
+		if writeExpected {
+			require.Equal(t, 1, len(ociMock.FlushCalls()), "%d: %v", i, tc)
+		} else {
+			require.Equal(t, 0, len(ociMock.FlushCalls()), "%d: %v", i, tc)
+		}
+	}
+}
+
+type MockShim struct {
+	called      bool
+	args        []string
+	returnError error
+}
+
+func (m *MockShim) Exec(args []string) error {
+	m.called = true
+	m.args = args
+	return m.returnError
+}

cmd/nvidia-container-runtime/runtime_factory.go

@@ -0,0 +1,74 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+const (
+	ociSpecFileName          = "config.json"
+	dockerRuncExecutableName = "docker-runc"
+	runcExecutableName       = "runc"
+)
+
+// newRuntime is a factory method that constructs a runtime based on the selected configuration.
+func newRuntime(argv []string) (oci.Runtime, error) {
+	ociSpec, err := newOCISpec(argv)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing OCI specification: %v", err)
+	}
+
+	runc, err := newRuncRuntime()
+	if err != nil {
+		return nil, fmt.Errorf("error constructing runc runtime: %v", err)
+	}
+
+	r, err := newNvidiaContainerRuntimeWithLogger(logger.Logger, runc, ociSpec)
+	if err != nil {
+		return nil, fmt.Errorf("error constructing NVIDIA Container Runtime: %v", err)
+	}
+
+	return r, nil
+}
+
+// newOCISpec constructs an OCI spec for the provided arguments
+func newOCISpec(argv []string) (oci.Spec, error) {
+	bundleDir, err := oci.GetBundleDir(argv)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing command line arguments: %v", err)
+	}
+	logger.Infof("Using bundle directory: %v", bundleDir)
+
+	ociSpecPath := oci.GetSpecFilePath(bundleDir)
+	logger.Infof("Using OCI specification file path: %v", ociSpecPath)
+
+	ociSpec := oci.NewSpecFromFile(ociSpecPath)
+
+	return ociSpec, nil
+}
+
+// newRuncRuntime locates the runc binary and wraps it in a SyscallExecRuntime
+func newRuncRuntime() (oci.Runtime, error) {
+	return oci.NewLowLevelRuntimeWithLogger(
+		logger.Logger,
+		dockerRuncExecutableName,
+		runcExecutableName,
+	)
+}

cmd/nvidia-container-runtime/runtime_factory_test.go

@@ -0,0 +1,30 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestConstructor(t *testing.T) {
+	shim, err := newRuntime([]string{})
+
+	require.NoError(t, err)
+	require.NotNil(t, shim)
+}

cmd/nvidia-container-toolkit/capabilities.go

@@ -0,0 +1,83 @@
+package main
+
+import (
+	"log"
+	"strings"
+)
+
+const (
+	allDriverCapabilities     = DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx")
+	defaultDriverCapabilities = DriverCapabilities("utility,compute")
+
+	none = DriverCapabilities("")
+	all  = DriverCapabilities("all")
+)
+
+func capabilityToCLI(cap string) string {
+	switch cap {
+	case "compute":
+		return "--compute"
+	case "compat32":
+		return "--compat32"
+	case "graphics":
+		return "--graphics"
+	case "utility":
+		return "--utility"
+	case "video":
+		return "--video"
+	case "display":
+		return "--display"
+	case "ngx":
+		return "--ngx"
+	default:
+		log.Panicln("unknown driver capability:", cap)
+	}
+	return ""
+}
+
+// DriverCapabilities is used to process the NVIDIA_DRIVER_CAPABILITIES environment
+// variable. Operations include default values, filtering, and handling meta values such as "all"
+type DriverCapabilities string
+
+// Intersection returns intersection between two sets of capabilities.
+func (d DriverCapabilities) Intersection(capabilities DriverCapabilities) DriverCapabilities {
+	if capabilities == all {
+		return d
+	}
+	if d == all {
+		return capabilities
+	}
+
+	lookup := make(map[string]bool)
+	for _, c := range d.list() {
+		lookup[c] = true
+	}
+	var found []string
+	for _, c := range capabilities.list() {
+		if lookup[c] {
+			found = append(found, c)
+		}
+	}
+
+	intersection := DriverCapabilities(strings.Join(found, ","))
+	return intersection
+}
+
+// String returns the string representation of the driver capabilities
+func (d DriverCapabilities) String() string {
+	return string(d)
+}
+
+// list returns the driver capabilities as a list
+func (d DriverCapabilities) list() []string {
+	var caps []string
+	for _, c := range strings.Split(string(d), ",") {
+		trimmed := strings.TrimSpace(c)
+		if len(trimmed) == 0 {
+			continue
+		}
+		caps = append(caps, trimmed)
+	}
+
+	return caps
+}

cmd/nvidia-container-toolkit/capabilities_test.go

@@ -0,0 +1,134 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestDriverCapabilitiesIntersection(t *testing.T) {
+	testCases := []struct {
+		capabilities          DriverCapabilities
+		supportedCapabilities DriverCapabilities
+		expectedIntersection  DriverCapabilities
+	}{
+		{
+			capabilities:          none,
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          all,
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          all,
+			supportedCapabilities: allDriverCapabilities,
+			expectedIntersection:  allDriverCapabilities,
+		},
+		{
+			capabilities:          allDriverCapabilities,
+			supportedCapabilities: all,
+			expectedIntersection:  allDriverCapabilities,
+		},
+		{
+			capabilities:          none,
+			supportedCapabilities: all,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          none,
+			supportedCapabilities: DriverCapabilities("cap1"),
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          DriverCapabilities("cap0,cap1"),
+			supportedCapabilities: DriverCapabilities("cap1,cap0"),
+			expectedIntersection:  DriverCapabilities("cap0,cap1"),
+		},
+		{
+			capabilities:          defaultDriverCapabilities,
+			supportedCapabilities: allDriverCapabilities,
+			expectedIntersection:  defaultDriverCapabilities,
+		},
+		{
+			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+		{
+			capabilities:          DriverCapabilities("cap1"),
+			supportedCapabilities: none,
+			expectedIntersection:  none,
+		},
+		{
+			capabilities:          DriverCapabilities("compute,compat32,graphics,utility,video,display,ngx"),
+			supportedCapabilities: DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+			expectedIntersection:  DriverCapabilities("compute,compat32,graphics,utility,video,display"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			intersection := tc.supportedCapabilities.Intersection(tc.capabilities)
+			require.EqualValues(t, tc.expectedIntersection, intersection)
+		})
+	}
+}
+
+func TestDriverCapabilitiesList(t *testing.T) {
+	testCases := []struct {
+		capabilities DriverCapabilities
+		expected     []string
+	}{
+		{
+			capabilities: DriverCapabilities(""),
+		},
+		{
+			capabilities: DriverCapabilities("  "),
+		},
+		{
+			capabilities: DriverCapabilities(","),
+		},
+		{
+			capabilities: DriverCapabilities(",cap"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: DriverCapabilities("cap,"),
+			expected:     []string{"cap"},
+		},
+		{
+			capabilities: DriverCapabilities("cap0,,cap1"),
+			expected:     []string{"cap0", "cap1"},
+		},
+		{
+			capabilities: DriverCapabilities("cap1,cap0,cap3"),
+			expected:     []string{"cap1", "cap0", "cap3"},
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			require.EqualValues(t, tc.expected, tc.capabilities.list())
+		})
+	}
+}

cmd/nvidia-container-toolkit/container_config.go

@@ -26,11 +26,6 @@ const (
 	envNVDriverCapabilities = "NVIDIA_DRIVER_CAPABILITIES"
 )
 
-const (
-	allDriverCapabilities     = "compute,compat32,graphics,utility,video,display,ngx"
-	defaultDriverCapabilities = "utility"
-)
-
 const (
 	capSysAdmin = "CAP_SYS_ADMIN"
 )
@@ -216,6 +211,7 @@ func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
 	for _, envVar := range envVars {
 		if devs, ok := env[envVar]; ok {
 			devices = &devs
+			break
 		}
 	}
 
@@ -295,8 +291,8 @@ func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, p
 		return devices
 	}
 
-	// Error out otherwise
-	log.Panicln("insufficient privileges to read device list from NVIDIA_VISIBLE_DEVICES envvar")
+	configName := hookConfig.getConfigOption("AcceptEnvvarUnprivileged")
+	log.Printf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES (privileged=%v, %v=%v) ", privileged, configName, hookConfig.AcceptEnvvarUnprivileged)
 
 	return nil
 }
@@ -315,33 +311,27 @@ func getMigMonitorDevices(env map[string]string) *string {
 	return nil
 }
 
-func getDriverCapabilities(env map[string]string, legacyImage bool) *string {
-	// Grab a reference to the capabilities from the envvar
-	// if it actually exists in the environment.
-	var capabilities *string
-	if caps, ok := env[envNVDriverCapabilities]; ok {
-		capabilities = &caps
+func getDriverCapabilities(env map[string]string, supportedDriverCapabilities DriverCapabilities, legacyImage bool) DriverCapabilities {
+	// We use the default driver capabilities by default. This is filtered to only include the
+	// supported capabilities
+	capabilities := supportedDriverCapabilities.Intersection(defaultDriverCapabilities)
+
+	capsEnv, capsEnvSpecified := env[envNVDriverCapabilities]
+
+	if !capsEnvSpecified && legacyImage {
+		// Environment variable unset with legacy image: set all capabilities.
+		return supportedDriverCapabilities
 	}
 
-	// Environment variable unset with legacy image: set all capabilities.
-	if capabilities == nil && legacyImage {
-		allCaps := allDriverCapabilities
-		return &allCaps
+	if capsEnvSpecified && len(capsEnv) > 0 {
+		// If the envvironment variable is specified and is non-empty, use the capabilities value
+		envCapabilities := DriverCapabilities(capsEnv)
+		capabilities = supportedDriverCapabilities.Intersection(envCapabilities)
+		if envCapabilities != all && capabilities != envCapabilities {
+			log.Panicln(fmt.Errorf("unsupported capabilities found in '%v' (allowed '%v')", envCapabilities, capabilities))
+		}
 	}
 
-	// Environment variable unset or set but empty: set default capabilities.
-	if capabilities == nil || len(*capabilities) == 0 {
-		defaultCaps := defaultDriverCapabilities
-		return &defaultCaps
-	}
-
-	// Environment variable set to "all": set all capabilities.
-	if *capabilities == "all" {
-		allCaps := allDriverCapabilities
-		return &allCaps
-	}
-
-	// Any other value
 	return capabilities
 }
 
@@ -388,10 +378,7 @@ func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mou
 		log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container")
 	}
 
-	var driverCapabilities string
-	if c := getDriverCapabilities(env, legacyImage); c != nil {
-		driverCapabilities = *c
-	}
+	driverCapabilities := getDriverCapabilities(env, hookConfig.SupportedDriverCapabilities, legacyImage).String()
 
 	requirements := getRequirements(env, legacyImage)
 

cmd/nvidia-container-toolkit/container_config_test.go

@@ -2,8 +2,9 @@ package main
 
 import (
 	"path/filepath"
-	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestGetNvidiaConfig(t *testing.T) {
@@ -11,6 +12,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 		description    string
 		env            map[string]string
 		privileged     bool
+		hookConfig     *HookConfig
 		expectedConfig *nvidiaConfig
 		expectedPanic  bool
 	}{
@@ -34,7 +36,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -48,7 +50,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -80,7 +82,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -94,7 +96,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -109,7 +111,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -124,7 +126,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -134,12 +136,12 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envCUDAVersion:          "9.0",
 				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "cap0,cap1",
+				envNVDriverCapabilities: "video,display",
 			},
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -149,14 +151,14 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envCUDAVersion:              "9.0",
 				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "cap0,cap1",
+				envNVDriverCapabilities:     "video,display",
 				envNVRequirePrefix + "REQ0": "req0=true",
 				envNVRequirePrefix + "REQ1": "req1=false",
 			},
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
 				DisableRequire:     false,
 			},
@@ -166,7 +168,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envCUDAVersion:              "9.0",
 				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "cap0,cap1",
+				envNVDriverCapabilities:     "video,display",
 				envNVRequirePrefix + "REQ0": "req0=true",
 				envNVRequirePrefix + "REQ1": "req1=false",
 				envNVDisableRequire:         "true",
@@ -174,7 +176,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
 				DisableRequire:     true,
 			},
@@ -205,7 +207,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -237,7 +239,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -251,7 +253,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -266,7 +268,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -281,7 +283,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: allDriverCapabilities,
+				DriverCapabilities: allDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -291,12 +293,12 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envNVRequireCUDA:        "cuda>=9.0",
 				envNVVisibleDevices:     "gpu0,gpu1",
-				envNVDriverCapabilities: "cap0,cap1",
+				envNVDriverCapabilities: "video,display",
 			},
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -306,14 +308,14 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envNVRequireCUDA:            "cuda>=9.0",
 				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "cap0,cap1",
+				envNVDriverCapabilities:     "video,display",
 				envNVRequirePrefix + "REQ0": "req0=true",
 				envNVRequirePrefix + "REQ1": "req1=false",
 			},
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
 				DisableRequire:     false,
 			},
@@ -323,7 +325,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			env: map[string]string{
 				envNVRequireCUDA:            "cuda>=9.0",
 				envNVVisibleDevices:         "gpu0,gpu1",
-				envNVDriverCapabilities:     "cap0,cap1",
+				envNVDriverCapabilities:     "video,display",
 				envNVRequirePrefix + "REQ0": "req0=true",
 				envNVRequirePrefix + "REQ1": "req1=false",
 				envNVDisableRequire:         "true",
@@ -331,7 +333,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged: false,
 			expectedConfig: &nvidiaConfig{
 				Devices:            "gpu0,gpu1",
-				DriverCapabilities: "cap0,cap1",
+				DriverCapabilities: "video,display",
 				Requirements:       []string{"cuda>=9.0", "req0=true", "req1=false"},
 				DisableRequire:     true,
 			},
@@ -345,7 +347,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{},
 				DisableRequire:     false,
 			},
@@ -361,7 +363,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
 				MigConfigDevices:   "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -387,7 +389,7 @@ func TestGetNvidiaConfig(t *testing.T) {
 			expectedConfig: &nvidiaConfig{
 				Devices:            "all",
 				MigMonitorDevices:  "mig0,mig1",
-				DriverCapabilities: defaultDriverCapabilities,
+				DriverCapabilities: defaultDriverCapabilities.String(),
 				Requirements:       []string{"cuda>=9.0"},
 				DisableRequire:     false,
 			},
@@ -402,19 +404,67 @@ func TestGetNvidiaConfig(t *testing.T) {
 			privileged:    false,
 			expectedPanic: true,
 		},
+		{
+			description: "Hook config set as driver-capabilities-all",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				envNVDriverCapabilities: "all",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: "video,display",
+			},
+		},
+		{
+			description: "Hook config set, envvar sets driver-capabilities",
+			env: map[string]string{
+				envNVVisibleDevices:     "all",
+				envNVDriverCapabilities: "video,display",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display,compute,utility",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: "video,display",
+			},
+		},
+		{
+			description: "Hook config set, envvar unset sets default driver-capabilities",
+			env: map[string]string{
+				envNVVisibleDevices: "all",
+			},
+			privileged: true,
+			hookConfig: &HookConfig{
+				SupportedDriverCapabilities: "video,display,utility,compute",
+			},
+			expectedConfig: &nvidiaConfig{
+				Devices:            "all",
+				DriverCapabilities: defaultDriverCapabilities.String(),
+			},
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			// Wrap the call to getNvidiaConfig() in a closure.
 			var config *nvidiaConfig
 			getConfig := func() {
-				hookConfig := getDefaultHookConfig()
-				config = getNvidiaConfig(&hookConfig, tc.env, nil, tc.privileged)
+				hookConfig := tc.hookConfig
+				if hookConfig == nil {
+					defaultConfig := getDefaultHookConfig()
+					hookConfig = &defaultConfig
+				}
+				config = getNvidiaConfig(hookConfig, tc.env, nil, tc.privileged)
 			}
 
 			// For any tests that are expected to panic, make sure they do.
 			if tc.expectedPanic {
-				mustPanic(t, getConfig)
+				require.Panics(t, getConfig)
 				return
 			}
 
@@ -422,31 +472,20 @@ func TestGetNvidiaConfig(t *testing.T) {
 			getConfig()
 
 			// And start comparing the test results to the expected results.
-			if config == nil && tc.expectedConfig == nil {
+			if tc.expectedConfig == nil {
+				require.Nil(t, config, tc.description)
 				return
 			}
-			if config != nil && tc.expectedConfig != nil {
-				if !reflect.DeepEqual(config.Devices, tc.expectedConfig.Devices) {
-					t.Errorf("Unexpected nvidiaConfig (got: %v, wanted: %v)", config, tc.expectedConfig)
-				}
-				if !reflect.DeepEqual(config.MigConfigDevices, tc.expectedConfig.MigConfigDevices) {
-					t.Errorf("Unexpected nvidiaConfig (got: %v, wanted: %v)", config, tc.expectedConfig)
-				}
-				if !reflect.DeepEqual(config.MigMonitorDevices, tc.expectedConfig.MigMonitorDevices) {
-					t.Errorf("Unexpected nvidiaConfig (got: %v, wanted: %v)", config, tc.expectedConfig)
-				}
-				if !reflect.DeepEqual(config.DriverCapabilities, tc.expectedConfig.DriverCapabilities) {
-					t.Errorf("Unexpected nvidiaConfig (got: %v, wanted: %v)", config, tc.expectedConfig)
-				}
-				if !elementsMatch(config.Requirements, tc.expectedConfig.Requirements) {
-					t.Errorf("Unexpected nvidiaConfig (got: %v, wanted: %v)", config, tc.expectedConfig)
-				}
-				if !reflect.DeepEqual(config.DisableRequire, tc.expectedConfig.DisableRequire) {
-					t.Errorf("Unexpected nvidiaConfig (got: %v, wanted: %v)", config, tc.expectedConfig)
-				}
-				return
-			}
-			t.Errorf("Unexpected nvidiaConfig (got: %v, wanted: %v)", config, tc.expectedConfig)
+
+			require.NotNil(t, config, tc.description)
+
+			require.Equal(t, tc.expectedConfig.Devices, config.Devices)
+			require.Equal(t, tc.expectedConfig.MigConfigDevices, config.MigConfigDevices)
+			require.Equal(t, tc.expectedConfig.MigMonitorDevices, config.MigMonitorDevices)
+			require.Equal(t, tc.expectedConfig.DriverCapabilities, config.DriverCapabilities)
+
+			require.ElementsMatch(t, tc.expectedConfig.Requirements, config.Requirements)
+			require.Equal(t, tc.expectedConfig.DisableRequire, config.DisableRequire)
 		})
 	}
 }
@@ -524,9 +563,7 @@ func TestGetDevicesFromMounts(t *testing.T) {
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
 			devices := getDevicesFromMounts(tc.mounts)
-			if !reflect.DeepEqual(devices, tc.expectedDevices) {
-				t.Errorf("Unexpected devices (got: %v, wanted: %v)", *devices, *tc.expectedDevices)
-			}
+			require.Equal(t, tc.expectedDevices, devices)
 		})
 	}
 }
@@ -540,7 +577,6 @@ func TestDeviceListSourcePriority(t *testing.T) {
 		acceptUnprivileged bool
 		acceptMounts       bool
 		expectedDevices    *string
-		expectedPanic      bool
 	}{
 		{
 			description: "Mount devices, unprivileged, no accept unprivileged",
@@ -567,7 +603,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
 			privileged:         false,
 			acceptUnprivileged: false,
 			acceptMounts:       true,
-			expectedPanic:      true,
+			expectedDevices:    nil,
 		},
 		{
 			description:        "No mount devices, privileged, no accept unprivileged",
@@ -621,7 +657,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
 			privileged:         false,
 			acceptUnprivileged: false,
 			acceptMounts:       false,
-			expectedPanic:      true,
+			expectedDevices:    nil,
 		},
 	}
 	for _, tc := range tests {
@@ -638,44 +674,316 @@ func TestDeviceListSourcePriority(t *testing.T) {
 				devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
 			}
 
-			// For any tests that are expected to panic, make sure they do.
-			if tc.expectedPanic {
-				mustPanic(t, getDevices)
-				return
-			}
-
 			// For all other tests, just grab the devices and check the results
 			getDevices()
-			if !reflect.DeepEqual(devices, tc.expectedDevices) {
-				t.Errorf("Unexpected devices (got: %v, wanted: %v)", *devices, *tc.expectedDevices)
-			}
+
+			require.Equal(t, tc.expectedDevices, devices)
 		})
 	}
 }
 
-func elementsMatch(slice0, slice1 []string) bool {
-	map0 := make(map[string]int)
-	map1 := make(map[string]int)
+func TestGetDevicesFromEnvvar(t *testing.T) {
+	all := "all"
+	empty := ""
+	envDockerResourceGPUs := "DOCKER_RESOURCE_GPUS"
+	gpuID := "GPU-12345"
+	anotherGPUID := "GPU-67890"
 
-	for _, e := range slice0 {
-		map0[e]++
+	var tests = []struct {
+		description     string
+		envSwarmGPU     *string
+		env             map[string]string
+		legacyImage     bool
+		expectedDevices *string
+	}{
+		{
+			description: "empty env returns nil for non-legacy image",
+		},
+		{
+			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: "",
+			},
+		},
+		{
+			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: "void",
+			},
+		},
+		{
+			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: "none",
+			},
+			expectedDevices: &empty,
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: gpuID,
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
+			env: map[string]string{
+				envNVVisibleDevices: gpuID,
+			},
+			legacyImage:     true,
+			expectedDevices: &gpuID,
+		},
+		{
+			description:     "empty env returns all for legacy image",
+			legacyImage:     true,
+			expectedDevices: &all,
+		},
+		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is ignored when
+		// not enabled
+		{
+			description: "missing NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envDockerResourceGPUs: anotherGPUID,
+			},
+		},
+		{
+			description: "blank NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   "",
+				envDockerResourceGPUs: anotherGPUID,
+			},
+		},
+		{
+			description: "'void' NVIDIA_VISIBLE_DEVICES returns nil for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   "void",
+				envDockerResourceGPUs: anotherGPUID,
+			},
+		},
+		{
+			description: "'none' NVIDIA_VISIBLE_DEVICES returns empty for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   "none",
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: &empty,
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for non-legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   gpuID,
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "NVIDIA_VISIBLE_DEVICES set returns value for legacy image",
+			env: map[string]string{
+				envNVVisibleDevices:   gpuID,
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			legacyImage:     true,
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "empty env returns all for legacy image",
+			env: map[string]string{
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			legacyImage:     true,
+			expectedDevices: &all,
+		},
+		// Add the `DOCKER_RESOURCE_GPUS` envvar and ensure that this is selected when
+		// enabled
+		{
+			description: "empty env returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+		},
+		{
+			description: "blank DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: "",
+			},
+		},
+		{
+			description: "'void' DOCKER_RESOURCE_GPUS returns nil for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: "void",
+			},
+		},
+		{
+			description: "'none' DOCKER_RESOURCE_GPUS returns empty for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: "none",
+			},
+			expectedDevices: &empty,
+		},
+		{
+			description: "DOCKER_RESOURCE_GPUS set returns value for non-legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: gpuID,
+			},
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "DOCKER_RESOURCE_GPUS set returns value for legacy image",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: gpuID,
+			},
+			legacyImage:     true,
+			expectedDevices: &gpuID,
+		},
+		{
+			description: "DOCKER_RESOURCE_GPUS is selected if present",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
+		{
+			description: "DOCKER_RESOURCE_GPUS overrides NVIDIA_VISIBLE_DEVICES if present",
+			envSwarmGPU: &envDockerResourceGPUs,
+			env: map[string]string{
+				envNVVisibleDevices:   gpuID,
+				envDockerResourceGPUs: anotherGPUID,
+			},
+			expectedDevices: &anotherGPUID,
+		},
 	}
 
-	for _, e := range slice1 {
-		map1[e]++
-	}
+	for i, tc := range tests {
+		t.Run(tc.description, func(t *testing.T) {
+			envSwarmGPU = tc.envSwarmGPU
+			devices := getDevicesFromEnvvar(tc.env, tc.legacyImage)
+			if tc.expectedDevices == nil {
+				require.Nil(t, devices, "%d: %v", i, tc)
+				return
+			}
 
-	for k0, v0 := range map0 {
-		if map1[k0] != v0 {
-			return false
-		}
+			require.NotNil(t, devices, "%d: %v", i, tc)
+			require.Equal(t, *tc.expectedDevices, *devices, "%d: %v", i, tc)
+		})
+	}
+}
+
+func TestGetDriverCapabilities(t *testing.T) {
+
+	supportedCapabilities := "compute,utility,display,video"
+
+	testCases := []struct {
+		description           string
+		env                   map[string]string
+		legacyImage           bool
+		supportedCapabilities string
+		expectedPanic         bool
+		expectedCapabilities  string
+	}{
+		{
+			description: "Env is set for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "display,video",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description: "Env is all for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "all",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for legacy image",
+			env: map[string]string{
+				envNVDriverCapabilities: "",
+			},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description:           "Env unset for legacy image is 'all'",
+			env:                   map[string]string{},
+			legacyImage:           true,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is set for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "display,video",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  "display,video",
+		},
+		{
+			description:           "Env unset for modern image is default",
+			env:                   map[string]string{},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description: "Env is all for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "all",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  supportedCapabilities,
+		},
+		{
+			description: "Env is empty for modern image",
+			env: map[string]string{
+				envNVDriverCapabilities: "",
+			},
+			legacyImage:           false,
+			supportedCapabilities: supportedCapabilities,
+			expectedCapabilities:  defaultDriverCapabilities.String(),
+		},
+		{
+			description: "Invalid capabilities panic",
+			env: map[string]string{
+				envNVDriverCapabilities: "compute,utility",
+			},
+			supportedCapabilities: "not-compute,not-utility",
+			expectedPanic:         true,
+		},
+		{
+			description:           "Default is restricted for modern image",
+			legacyImage:           false,
+			supportedCapabilities: "compute",
+			expectedCapabilities:  "compute",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			var capabilites DriverCapabilities
+
+			getDriverCapabilities := func() {
+				supportedCapabilities := DriverCapabilities(tc.supportedCapabilities)
+				capabilites = getDriverCapabilities(tc.env, supportedCapabilities, tc.legacyImage)
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getDriverCapabilities)
+				return
+			}
+
+			getDriverCapabilities()
+			require.EqualValues(t, tc.expectedCapabilities, capabilites)
+		})
 	}
-
-	for k1, v1 := range map1 {
-		if map0[k1] != v1 {
-			return false
-		}
-	}
-
-	return true
 }

cmd/nvidia-container-toolkit/hook_config.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"os"
 	"path"
+	"reflect"
 
 	"github.com/BurntSushi/toml"
 )
@@ -34,10 +35,11 @@ type CLIConfig struct {
 
 // HookConfig : options for the nvidia-container-toolkit.
 type HookConfig struct {
-	DisableRequire                 bool    `toml:"disable-require"`
-	SwarmResource                  *string `toml:"swarm-resource"`
-	AcceptEnvvarUnprivileged       bool    `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
-	AcceptDeviceListAsVolumeMounts bool    `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
+	DisableRequire                 bool               `toml:"disable-require"`
+	SwarmResource                  *string            `toml:"swarm-resource"`
+	AcceptEnvvarUnprivileged       bool               `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
+	AcceptDeviceListAsVolumeMounts bool               `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
+	SupportedDriverCapabilities    DriverCapabilities `toml:"supported-driver-capabilities"`
 
 	NvidiaContainerCLI CLIConfig `toml:"nvidia-container-cli"`
 }
@@ -48,6 +50,7 @@ func getDefaultHookConfig() (config HookConfig) {
 		SwarmResource:                  nil,
 		AcceptEnvvarUnprivileged:       true,
 		AcceptDeviceListAsVolumeMounts: false,
+		SupportedDriverCapabilities:    allDriverCapabilities,
 		NvidiaContainerCLI: CLIConfig{
 			Root:        nil,
 			Path:        nil,
@@ -84,5 +87,29 @@ func getHookConfig() (config HookConfig) {
 		}
 	}
 
+	if config.SupportedDriverCapabilities == all {
+		config.SupportedDriverCapabilities = allDriverCapabilities
+	}
+	// We ensure that the supported-driver-capabilites option is a subset of allDriverCapabilities
+	if intersection := allDriverCapabilities.Intersection(config.SupportedDriverCapabilities); intersection != config.SupportedDriverCapabilities {
+		configName := config.getConfigOption("SupportedDriverCapabilities")
+		log.Panicf("Invalid value for config option '%v'; %v (supported: %v)\n", configName, config.SupportedDriverCapabilities, allDriverCapabilities)
+	}
+
 	return config
 }
+
+// getConfigOption returns the toml config option associated with the
+// specified struct field.
+func (c HookConfig) getConfigOption(fieldName string) string {
+	t := reflect.TypeOf(c)
+	f, ok := t.FieldByName(fieldName)
+	if !ok {
+		return fieldName
+	}
+	v, ok := f.Tag.Lookup("toml")
+	if !ok {
+		return fieldName
+	}
+	return v
+}

cmd/nvidia-container-toolkit/hook_config_test.go

@@ -0,0 +1,105 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package main
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetHookConfig(t *testing.T) {
+	testCases := []struct {
+		lines                      []string
+		expectedPanic              bool
+		expectedDriverCapabilities DriverCapabilities
+	}{
+		{
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"all\"",
+			},
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"compute,utility,not-compute\"",
+			},
+			expectedPanic: true,
+		},
+		{
+			lines:                      []string{},
+			expectedDriverCapabilities: allDriverCapabilities,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"\"",
+			},
+			expectedDriverCapabilities: none,
+		},
+		{
+			lines: []string{
+				"supported-driver-capabilities = \"utility,compute\"",
+			},
+			expectedDriverCapabilities: DriverCapabilities("utility,compute"),
+		},
+	}
+
+	for i, tc := range testCases {
+		t.Run(fmt.Sprintf("test case %d", i), func(t *testing.T) {
+			var filename string
+			defer func() {
+				if len(filename) > 0 {
+					os.Remove(filename)
+				}
+				configflag = nil
+			}()
+
+			if tc.lines != nil {
+				configFile, err := os.CreateTemp("", "*.toml")
+				require.NoError(t, err)
+				defer configFile.Close()
+
+				filename = configFile.Name()
+				configflag = &filename
+
+				for _, line := range tc.lines {
+					_, err := configFile.WriteString(fmt.Sprintf("%s\n", line))
+					require.NoError(t, err)
+				}
+			}
+
+			var config HookConfig
+			getHookConfig := func() {
+				config = getHookConfig()
+			}
+
+			if tc.expectedPanic {
+				require.Panics(t, getHookConfig)
+				return
+			}
+
+			getHookConfig()
+
+			require.EqualValues(t, tc.expectedDriverCapabilities, config.SupportedDriverCapabilities)
+		})
+	}
+}

cmd/nvidia-container-toolkit/hook_test.go

@@ -3,6 +3,8 @@ package main
 import (
 	"encoding/json"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 func TestParseCudaVersionValid(t *testing.T) {
@@ -16,24 +18,15 @@ func TestParseCudaVersionValid(t *testing.T) {
 		{"9.0.116", [3]uint32{9, 0, 116}},
 		{"4294967295.4294967295.4294967295", [3]uint32{4294967295, 4294967295, 4294967295}},
 	}
-	for _, c := range tests {
+	for i, c := range tests {
 		vmaj, vmin, vpatch := parseCudaVersion(c.version)
-		if vmaj != c.expected[0] || vmin != c.expected[1] || vpatch != c.expected[2] {
-			t.Errorf("parseCudaVersion(%s): %d.%d.%d (expected: %v)", c.version, vmaj, vmin, vpatch, c.expected)
-		}
+
+		version := [3]uint32{vmaj, vmin, vpatch}
+
+		require.Equal(t, c.expected, version, "%d: %v", i, c)
 	}
 }
 
-func mustPanic(t *testing.T, f func()) {
-	defer func() {
-		if err := recover(); err == nil {
-			t.Error("Test didn't panic!")
-		}
-	}()
-
-	f()
-}
-
 func TestParseCudaVersionInvalid(t *testing.T) {
 	var tests = []string{
 		"foo",
@@ -53,10 +46,9 @@ func TestParseCudaVersionInvalid(t *testing.T) {
 		"-9.-1.-116",
 	}
 	for _, c := range tests {
-		mustPanic(t, func() {
-			t.Logf("parseCudaVersion(%s)", c)
+		require.Panics(t, func() {
 			parseCudaVersion(c)
-		})
+		}, "parseCudaVersion(%v)", c)
 	}
 }
 
@@ -132,12 +124,11 @@ func TestIsPrivileged(t *testing.T) {
 			false,
 		},
 	}
-	for _, tc := range tests {
+	for i, tc := range tests {
 		var spec Spec
 		_ = json.Unmarshal([]byte(tc.spec), &spec)
 		privileged := isPrivileged(&spec)
-		if privileged != tc.expected {
-			t.Errorf("isPrivileged() returned unexpectred value (privileged: %v, tc.expected: %v)", privileged, tc.expected)
-		}
+
+		require.Equal(t, tc.expected, privileged, "%d: %v", i, tc)
 	}
 }

config/config.toml.ubuntu+jetpack

@@ -0,0 +1,19 @@
+disable-require = false
+supported-driver-capabilities = "compute,compat32,graphics,utility,video,display"
+#swarm-resource = "DOCKER_RESOURCE_GPU"
+#accept-nvidia-visible-devices-envvar-when-unprivileged = true
+#accept-nvidia-visible-devices-as-volume-mounts = false
+
+[nvidia-container-cli]
+#root = "/run/nvidia/driver"
+#path = "/usr/bin/nvidia-container-cli"
+environment = []
+#debug = "/var/log/nvidia-container-toolkit.log"
+#ldcache = "/etc/ld.so.cache"
+load-kmods = true
+#no-cgroups = false
+#user = "root:video"
+ldconfig = "@/sbin/ldconfig.real"
+
+[nvidia-container-runtime]
+#debug = "/var/log/nvidia-container-runtime.log"

docker/Dockerfile.amazonlinux

@@ -40,10 +40,11 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.amzn $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
 # This might not be useful on Amazon Linux, but it's simpler to keep the RHEL
@@ -60,6 +61,7 @@ CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
              -D "version $VERSION" \
+             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
              -D "release $RELEASE" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.centos

@@ -40,10 +40,11 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.centos $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
 COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
@@ -58,6 +59,7 @@ CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
              -D "version $VERSION" \
+             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
              -D "release $RELEASE" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.debian

@@ -48,10 +48,11 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.debian $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 # Debian Jessie still had ldconfig.real
 RUN if [ "$(lsb_release -cs)" = "jessie" ]; then \
@@ -62,8 +63,11 @@ WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
 RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
+    dch --changelog debian/changelog -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/nvidia-container-toolkit_*.deb /dist

docker/Dockerfile.devel

@@ -0,0 +1,20 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+ARG GOLANG_VERSION=x.x.x
+FROM golang:${GOLANG_VERSION}
+
+RUN go get -u golang.org/x/lint/golint
+RUN go get -u github.com/matryer/moq
+RUN go get -u github.com/gordonklaus/ineffassign
+RUN go get -u github.com/client9/misspell/cmd/misspell

docker/Dockerfile.opensuse-leap

@@ -39,8 +39,7 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+RUN make PREFIX=${DIST_DIR} cmds
 
 # Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch
 COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
@@ -48,7 +47,9 @@ COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook
 # Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md
 COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json
 
-COPY config/config.toml.opensuse-leap $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 WORKDIR $DIST_DIR/..
 COPY packaging/rpm .
@@ -57,6 +58,7 @@ CMD arch=$(uname -m) && \
     rpmbuild --clean --target=$arch -bb \
              -D "_topdir $PWD" \
              -D "version $VERSION" \
+             -D "libnvidia_container_version ${VERSION}-${RELEASE}" \
              -D "release $RELEASE" \
              SPECS/nvidia-container-toolkit.spec && \
     mv RPMS/$arch/*.rpm /dist

docker/Dockerfile.ubuntu

@@ -46,17 +46,21 @@ RUN mkdir -p $DIST_DIR /dist
 WORKDIR $GOPATH/src/nvidia-container-toolkit
 COPY . .
 
-RUN make binary && \
-    mv ./nvidia-container-toolkit $DIST_DIR/nvidia-container-toolkit
+RUN make PREFIX=${DIST_DIR} cmds
 
-COPY config/config.toml.ubuntu $DIST_DIR/config.toml
+ARG CONFIG_TOML_SUFFIX
+ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX}
+COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml
 
 WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
 RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
+    dch --changelog debian/changelog -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \
-    debuild -eDISTRIB -eSECTION --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
+    debuild -eDISTRIB -eSECTION -eLIBNVIDIA_CONTAINER_VERSION="${REVISION}" \
+    --dpkg-buildpackage-hook='sh debian/prepare' -i -us -uc -b && \
     mv /tmp/*.deb /dist

docker/docker.mk

@@ -1,11 +1,23 @@
-# Copyright (c) 2017-2020, NVIDIA CORPORATION. All rights reserved.
+# Copyright (c) 2017-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 
 # Supported OSs by architecture
 AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9
 X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux1 amazonlinux2 opensuse-leap15.1
 PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8
 ARM64_TARGETS := ubuntu20.04 ubuntu18.04
-AARCH64_TARGETS := centos8 rhel8
+AARCH64_TARGETS := centos8 rhel8 amazonlinux2
 
 # Define top-level build targets
 docker%: SHELL:=/bin/bash
@@ -85,11 +97,11 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 
 # private centos target
 --centos%: OS := centos
---centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),2)
+--centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private amazonlinux target
 --amazonlinux%: OS := amazonlinux
---amazonlinux%: PKG_REV = $(if $(LIB_TAG),0.1.$(LIB_TAG).amzn$(VERSION),2.amzn$(VERSION))
+--amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 # private opensuse-leap target
 --opensuse-leap%: OS = opensuse-leap
@@ -98,20 +110,24 @@ docker-all: $(AMD64_TARGETS) $(X86_64_TARGETS) \
 
 # private rhel target (actually built on centos)
 --rhel%: OS := centos
---rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),2)
+--rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM))
 --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH)
 
+# We allow the CONFIG_TOML_SUFFIX to be overridden.
+CONFIG_TOML_SUFFIX ?= $(OS)
+
 docker-build-%:
 	@echo "Building for $(TARGET_PLATFORM)"
 	docker pull --platform=linux/$(ARCH) $(BASEIMAGE)
 	DOCKER_BUILDKIT=1 \
 	$(DOCKER) build \
 	    --progress=plain \
-	    --build-arg BASEIMAGE=$(BASEIMAGE) \
+	    --build-arg BASEIMAGE="$(BASEIMAGE)" \
 	    --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
 	    --build-arg PKG_VERS="$(LIB_VERSION)" \
 	    --build-arg PKG_REV="$(PKG_REV)" \
+	    --build-arg CONFIG_TOML_SUFFIX="$(CONFIG_TOML_SUFFIX)" \
 	    --tag $(BUILDIMAGE) \
 	    --file $(DOCKERFILE) .
 	$(DOCKER) run \

go.mod

@@ -4,6 +4,13 @@ go 1.14
 
 require (
 	github.com/BurntSushi/toml v0.3.1
-	github.com/stretchr/testify v1.6.0
+	github.com/containers/podman/v2 v2.2.1
+	github.com/opencontainers/runtime-spec v1.0.3-0.20211101234015-a3c33d663ebc
+	github.com/pelletier/go-toml v1.9.3
+	github.com/sirupsen/logrus v1.8.1
+	github.com/stretchr/testify v1.7.0
+	github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d
+	github.com/urfave/cli/v2 v2.3.0
 	golang.org/x/mod v0.3.0
+	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
 )

go.sum

@@ -1,25 +1,741 @@
+bazil.org/fuse v0.0.0-20160811212531-371fbbdaa898/go.mod h1:Xbm+BRKSBEpa4q4hTSxohYNQpsxXPbPry4JJWOB3LB8=
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU=
+cloud.google.com/go v0.44.1/go.mod h1:iSa0KzasP4Uvy3f1mN/7PiObzGgflwredwwASm/v6AU=
+cloud.google.com/go v0.44.2/go.mod h1:60680Gw3Yr4ikxnPRS/oxxkBccT6SA1yMk63TGekxKY=
+cloud.google.com/go v0.45.1/go.mod h1:RpBamKRgapWJb87xiFSdk4g1CME7QZg3uwTez+TSTjc=
+cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg0=
+cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
+cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
+cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
+cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
+cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
+dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/14rcole/gopopulate v0.0.0-20180821133914-b175b219e774/go.mod h1:6/0dYRLLXyJjbkIPeeGyoJ/eKOSI0eU6eTlCBYibgd0=
+github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8=
+github.com/Azure/go-autorest v11.1.2+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
+github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA=
+github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.15-0.20200113171025-3fe6c5262873/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/go-winio v0.4.15/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw=
+github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
+github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8=
+github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/PuerkitoBio/purell v1.0.0/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0=
+github.com/PuerkitoBio/urlesc v0.0.0-20160726150825-5bd2802263f2/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE=
+github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
+github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alexflint/go-filemutex v0.0.0-20171022225611-72bdc8eae2ae/go.mod h1:CgnQgUtFrFz9mxFNtED3jI5tLDjKlOM+oUF/sTk6ps0=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
+github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk=
+github.com/buger/goterm v0.0.0-20181115115552-c206103e1f37/go.mod h1:u9UyCz2eTrSGy6fbupqJ54eY5c4IC8gREQ1053dK12U=
+github.com/buger/jsonparser v0.0.0-20180808090653-f4dd9f5a6b44/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b/go.mod h1:TrMrLQfeENAPYPRsJuq3jsqdlRh3lvi6trTZJG8+tho=
+github.com/checkpoint-restore/go-criu/v4 v4.0.2/go.mod h1:xUQBLp4RLc5zJtWY++yjOoMoB5lihDt7fai+75m+rGw=
+github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
+github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
+github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
+github.com/cilium/ebpf v0.0.0-20200507155900-a9f01edf17e3/go.mod h1:XT+cAw5wfvsodedcijoh1l9cf7v1x9FlFB/3VmF/O8s=
+github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLIdUjrmSXlK9pkrsDlLHbO8jiB8X8JnOc=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
+github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM=
+github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko=
+github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw=
+github.com/containerd/console v1.0.0/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE=
+github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
+github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb/go.mod h1:Dq467ZllaHgAtVp4p1xUQWBrFXR9s/wyoTpG8zOJGkY=
+github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
+github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0=
+github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o=
+github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc=
+github.com/containernetworking/cni v0.7.2-0.20190904153231-83439463f784/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY=
+github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0=
+github.com/containers/buildah v1.18.0/go.mod h1:qHLk7RUL7cHfA7ve1MKkZ6cyKUxHD0YxiLJcKY+mJe8=
+github.com/containers/common v0.26.3/go.mod h1:hJWZIlrl5MsE2ELNRa+MPp6I1kPbXHauuj0Ym4BsLG4=
+github.com/containers/common v0.29.0/go.mod h1:yT4GTUHsKRmpaDb+mecXRnIMre7W3ZgwXqaYMywXlaA=
+github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I=
+github.com/containers/image/v5 v5.7.0/go.mod h1:8aOy+YaItukxghRORkvhq5ibWttHErzDLy6egrKfKos=
+github.com/containers/image/v5 v5.8.0/go.mod h1:jKxdRtyIDumVa56hdsZvV+gwx4zB50hRou6pIuCWLkg=
+github.com/containers/image/v5 v5.8.1/go.mod h1:blOEFd/iFdeyh891ByhCVUc+xAcaI3gBegXECwz9UbQ=
+github.com/containers/image/v5 v5.9.0/go.mod h1:blOEFd/iFdeyh891ByhCVUc+xAcaI3gBegXECwz9UbQ=
+github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY=
+github.com/containers/ocicrypt v1.0.3/go.mod h1:CUBa+8MRNL/VkpxYIpaMtgn1WgXGyvPQj8jcy0EVG6g=
+github.com/containers/podman/v2 v2.2.1 h1:ONDOHuzYnPF+ZJ+sV9hjtssAG93gTbyvlKN0LhbFIRY=
+github.com/containers/podman/v2 v2.2.1/go.mod h1:4CuPT3c5jB1XxIjFRiAkqrvXrW+g5NR5wQb58u4KJE0=
+github.com/containers/psgo v1.5.1/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzPUWfawVU=
+github.com/containers/storage v1.23.6/go.mod h1:haFs0HRowKwyzvWEx9EgI3WsL8XCSnBDb5f8P5CAxJY=
+github.com/containers/storage v1.23.7/go.mod h1:cUT2zHjtx+WlVri30obWmM2gpqpi8jfPsmIzP1TVpEI=
+github.com/containers/storage v1.24.0/go.mod h1:A4d3BzuZK9b3oLVEsiSRhZLPIx3z7utgiPyXLK/YMhY=
+github.com/containers/storage v1.24.1/go.mod h1:0xJL06Dmd+ZYXIUdnBUPN0JnhHGgwMkLvnnAonJfWJU=
+github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
+github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
+github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU=
+github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
+github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk=
+github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA=
+github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk=
+github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE=
+github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM=
+github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
+github.com/cri-o/ocicni v0.2.1-0.20201102180012-75c612fda1a2/go.mod h1:vingr1ztOAzP2WyTgGbpMov9dFhbjNxdLtDv0+PhAvY=
+github.com/cyphar/filepath-securejoin v0.2.2/go.mod h1:FpkQEhXnPnOthhzymB7CGsFk2G9VLXONKD9G7QGMM+4=
+github.com/d2g/dhcp4 v0.0.0-20170904100407-a1d1b6c41b1c/go.mod h1:Ct2BUK8SB0YC1SMSibvLzxjeJLnrYEVLULFNiHY9YfQ=
+github.com/d2g/dhcp4client v1.0.0/go.mod h1:j0hNfjhrt2SxUOw55nL0ATM/z4Yt3t2Kd1mW34z5W5s=
+github.com/d2g/dhcp4server v0.0.0-20181031114812-7d4a0a7f59a5/go.mod h1:Eo87+Kg/IX2hfWJfwxMzLyuSZyxSoAug2nGa1G2QAi8=
+github.com/d2g/hardwareaddr v0.0.0-20190221164911-e7d9fbe030e4/go.mod h1:bMl4RjIciD2oAxI7DmWRx6gbeqrkoLqv3MV0vzNad+I=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgrijalva/jwt-go v0.0.0-20160705203006-01aeca54ebda/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
+github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
+github.com/docker/docker v1.4.2-0.20191219165747-a9416c67da9f/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v17.12.0-ce-rc1.0.20200505174321-1655290016ac+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker v17.12.0-ce-rc1.0.20201020191947-73dc6a680cdd+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y=
+github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
+github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw=
+github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8=
+github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
+github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
+github.com/elazarl/goproxy v0.0.0-20170405201442-c4fc26588b6e/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/elazarl/goproxy v0.0.0-20180725130230-947c36da3153/go.mod h1:/Zj4wYkgs4iZTTu3o/KG3Itv/qCCa8VVMlb3i9OVuzc=
+github.com/emicklei/go-restful v0.0.0-20170410110728-ff4f55a20633/go.mod h1:otzb+WCGbkyDHkqmQmT5YD2WR4BBwUdeQoFo8l/7tVs=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/evanphx/json-patch v0.0.0-20190203023257-5858425f7550/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
+github.com/fsouza/go-dockerclient v1.6.6/go.mod h1:3/oRIWoe7uT6bwtAayj/EmJmepBjeL4pYvt7ZxC7Rnk=
+github.com/ghodss/yaml v0.0.0-20150909031657-73d445a93680/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
+github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas=
+github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU=
+github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8=
+github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0=
+github.com/go-openapi/jsonreference v0.0.0-20160704190145-13c6e3589ad9/go.mod h1:W3Z9FmVs9qj+KR4zFKmDPGiLdk1D9Rlm7cyMvf57TTg=
+github.com/go-openapi/spec v0.0.0-20160808142527-6aced65f8501/go.mod h1:J8+jY1nAiCcj+friV/PDoE1/3eeccG9LYBs0tYvLOWc=
+github.com/go-openapi/swag v0.0.0-20160704191624-1d0bd113de87/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I=
+github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
+github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
+github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
+github.com/gogo/protobuf v0.0.0-20171007142547-342cbe0a0415/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
+github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
+github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/google/btree v0.0.0-20160524151835-7d79101e329e/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI=
+github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
+github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
+github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
+github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY=
+github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg=
+github.com/gophercloud/gophercloud v0.0.0-20190126172459-c818fa66e4c8/go.mod h1:3WdhXV3rUYy9p6AUW8d94kr+HS62Y4VL9mBnFxsD8q4=
+github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
+github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
+github.com/gorilla/schema v1.2.0/go.mod h1:kgLaKoK1FELgZqMAVxx/5cbj0kT+57qxUrAlIO2eleU=
+github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gregjones/httpcache v0.0.0-20170728041850-787624de3eb7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA=
+github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
+github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q=
+github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
+github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
+github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
+github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
+github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
+github.com/insomniacslk/dhcp v0.0.0-20200806210722-3f14f7f8bd9c/go.mod h1:CfMdguCK66I5DAUJgGKyNz8aB6vO5dZzkm9Xep6WGvw=
+github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg=
+github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA=
+github.com/jamescun/tuntap v0.0.0-20190712092105-cb1fb277045c/go.mod h1:zzwpsgcYhzzIP5WyF8g9ivCv38cY9uAV9Gu0m3lThhE=
+github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks=
+github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
+github.com/json-iterator/go v0.0.0-20180701071628-ab8a2e0c74be/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
+github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
+github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
+github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a/go.mod h1:UJSiEoRfvx3hP73CvoARgeLjaIOjybY9vj8PUPPFGeU=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=
+github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
+github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.11.1/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.2/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI=
+github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
+github.com/mailru/easyjson v0.0.0-20160728113105-d5b7844b561a/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
+github.com/manifoldco/promptui v0.8.0/go.mod h1:n4zTdgP0vr0S3w7/O/g98U+e0gwLScEXGwov2nIKuGQ=
+github.com/martini-contrib/render v0.0.0-20150707142108-ec18f8345a11/go.mod h1:Ah2dBMoxZEqk118as2T4u4fjfXarE0pPnMJaArZQZsI=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
+github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o=
+github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
+github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
+github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4=
+github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
+github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
+github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
+github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/moby/sys/mount v0.1.0/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74=
+github.com/moby/sys/mount v0.1.1/go.mod h1:FVQFLDRWwyBjDTBNQXDlWnSFREqOo3OKX9aqhmeoo74=
+github.com/moby/sys/mountinfo v0.1.0/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
+github.com/moby/sys/mountinfo v0.1.3/go.mod h1:w2t2Avltqx8vE7gX5l+QiBKxODu2TX0+Syr3h52Tw4o=
+github.com/moby/sys/mountinfo v0.3.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A=
+github.com/moby/term v0.0.0-20200429084858-129dac9f73f6/go.mod h1:or9wGItza1sRcM4Wd3dIv8DsFHYQuFsMHEdxUIlUxms=
+github.com/moby/term v0.0.0-20200915141129-7f0af18e79f2/go.mod h1:TjQg8pa4iejrUrjiz0MCtMV38jdMNW4doKSiBrEvCQQ=
+github.com/moby/vpnkit v0.4.0/go.mod h1:KyjUrL9cb6ZSNNAUwZfqRjhwwgJ3BJN+kXh0t43WTUQ=
+github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618/go.mod h1:x8F1gnqOkIEiO4rqoeEEEqQbo7HjGMTvyoq3gej4iT0=
+github.com/mtrmac/gpgme v0.1.2/go.mod h1:GYYHnGSuS7HK3zVS2n3y73y0okK/BeKzwnn5jgiVFNI=
+github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs=
+github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=
+github.com/nlopes/slack v0.6.0/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk=
+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=
+github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
+github.com/onsi/ginkgo v0.0.0-20151202141238-7f8ab55aaf3b/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v0.0.0-20170829012221-11459a886d9c/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.10.1/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
+github.com/onsi/ginkgo v1.14.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY=
+github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
+github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
+github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc=
+github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
+github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v0.0.0-20190425234816-dae70e8efea4/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U=
+github.com/opencontainers/runc v1.0.0-rc91/go.mod h1:3Sm6Dt7OT8z88EbdQqqcRN2oCT54jbi72tT/HqgflT8=
+github.com/opencontainers/runc v1.0.0-rc91.0.20200708210054-ce54a9d4d79b/go.mod h1:ZuXhqlr4EiRYgDrBDNfSbE4+n9JX4+V107NwAmF7sZA=
+github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20200520003142-237cc4f519e2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20200710190001-3e4195d92445/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20200817204227-f9c09b4ea1df/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-spec v1.0.3-0.20211101234015-a3c33d663ebc h1:Q4P71bRTYvUK/qLX897YrMBA1oznb1noH0rLB5ji9SE=
+github.com/opencontainers/runtime-spec v1.0.3-0.20211101234015-a3c33d663ebc/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
+github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs=
+github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g=
+github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE=
+github.com/openshift/imagebuilder v1.1.8/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo=
+github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
+github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913/go.mod h1:J6OG6YJVEWopen4avK3VNQSnALmmjvniMmni/YFYAwc=
+github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
+github.com/pelletier/go-toml v1.9.3 h1:zeC5b1GviRUyKYd6OJPvBU/mcVDVoL1OhT17FCt5dSQ=
+github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c=
+github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU=
+github.com/pkg/errors v0.0.0-20190227000051-27936f6d90f9/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
+github.com/pquerna/ffjson v0.0.0-20190813045741-dac163c6c0a9/go.mod h1:YARuvh7BUWHNhzDq2OM5tzR2RiCcN2D7sapiKyCel/M=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
+github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
+github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
+github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
+github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
+github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
+github.com/rootless-containers/rootlesskit v0.11.1/go.mod h1:pCUqFJBGOIonbjQBaxSVnk3w3KnK2drqjllgpgvNnO8=
+github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo=
+github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
+github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
+github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo=
+github.com/seccomp/libseccomp-golang v0.9.2-0.20200616122406-847368b35ebf/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg=
+github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
+github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
+github.com/sirupsen/logrus v0.0.0-20190403091019-9b3cdde74fbe/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
+github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE=
+github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
+github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
+github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
+github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
+github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.2-0.20171109065643-2da4a54c5cee/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
+github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI=
+github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=
+github.com/spf13/pflag v0.0.0-20170130214245-9ff6c6923cff/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s=
+github.com/spf13/viper v1.4.0/go.mod h1:PTJ7Z/lr49W6bUbkmS1V3by4uWynFiR9p7+dSq/yZzE=
+github.com/spf13/viper v1.7.0/go.mod h1:8WkrPz2fc9jxqZNCJI/76HCieCp4Q8HaLFoCha5qpdg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho=
-github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
+github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww=
+github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I=
+github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
+github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d h1:hq9X/cf03C5rCx9yWhY7eMHiNxmhTMJAc5DQBq9BfnI=
+github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d/go.mod h1:oFPCwcQpP90RVZxlBdgPN+iu2tPkboPUa4xaVEI6pO4=
+github.com/tsaikd/govalidator v0.0.0-20161031084447-986f2244fc69/go.mod h1:yJymgtZhuWi1Ih5t37Ej381BGZFZvlb9YMTwBxB/QjU=
+github.com/u-root/u-root v6.0.0+incompatible/go.mod h1:RYkpo8pTHrNjW08opNd/U6p/RJE7K0D8fXO0d47+3YY=
+github.com/uber/jaeger-client-go v2.25.0+incompatible/go.mod h1:WVhlPFC8FDjOFMMWRy2pZqQJSXxYSwNYOkTr/Z6d3Kk=
+github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6+uUTzImX/AauajbLI56U=
+github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
+github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
+github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14=
+github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
+github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
+github.com/urfave/cli/v2 v2.3.0 h1:qph92Y649prgesehzOrQjdWyxFOp/QVM+6imKHad91M=
+github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI=
+github.com/varlink/go v0.0.0-20190502142041-0f1d566d194b/go.mod h1:YHaw8N660ESgMgLOZfLQqT1htFItynAUxMesFBho52s=
+github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g=
+github.com/vbauerster/mpb/v5 v5.3.0/go.mod h1:4yTkvAb8Cm4eylAp6t0JRq6pXDkFJ4krUlDqWYkakAs=
+github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk=
+github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE=
+github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmFBXmLKZu9Nxj3WKYEafiSqer2rnvPr0en9UNpI=
+github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU=
+github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonpointer v0.0.0-20190809123943-df4f5c81cb3b/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
+github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
+github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
+go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ=
+go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk=
+go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
+go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8=
+go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
+go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
+go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
+golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181025213731-e84da0312774/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200429183012-4b2356b1ed79/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
+golang.org/x/exp v0.0.0-20190829153037-c13cbed26979/go.mod h1:86+5VVa7VpoJ4kLfm080zCjGlMRFzhUhsZKEZO7MGek=
+golang.org/x/exp v0.0.0-20191030013958-a1ab85dbe136/go.mod h1:JXzH8nQsPlswgeRAPE3MuO9GYsAcnJvJ4vnMwN/5qkY=
+golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
+golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE=
+golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o=
+golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc=
+golang.org/x/mod v0.1.0/go.mod h1:0QHyrYULN0/3qlju5TqG8bIK38QM8yzMo5ekMj3DlcY=
 golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4=
 golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181011144130-49bb7cea24b1/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190206173232-65e2d4e15006/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
+golang.org/x/net v0.0.0-20201006153459-a7d1128ccaa0/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190425145619-16072639606e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200810151505-1b9f1253b3ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201018230417-eeed37f84f13/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887 h1:dXfMednGJh/SUUFjTLsWJz3P+TQt9qnR11GgeI3vWKs=
+golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.1-0.20181227161524-e6919f6577db/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/time v0.0.0-20161028155119-f51c12702a4d/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
+golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181011042414-1f849cf54d09/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312151545-0bb0c0a6e846/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190624222133-a101b041ded4/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190628153133-6cdbf07be9d0/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
+golang.org/x/tools v0.0.0-20190816200558-6889da9d5479/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20190911174233-4f2ddba30aff/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191001123449-8b695b21ef34/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191012152004-8de300cfc20a/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191112195655-aa38f8e97acc/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
+google.golang.org/api v0.7.0/go.mod h1:WtwebWUNSVBH/HAw79HIFXZNqEvBhG+Ra+ax0hx3E3M=
+google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.10.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
+google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
+google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
+google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
+google.golang.org/genproto v0.0.0-20191108220845-16a3f7862a1a/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
+google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU=
+gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=
+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=
+gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2/go.mod h1:Xk6kEKp8OKb+X14hQBKWaSkCsqBpgog8nAV2xsGOxlo=
+gopkg.in/inf.v0 v0.9.0/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw=
+gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
+gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776 h1:tQIYjPdBoyREyB9XMu+nnTclpTYkz2zFM+lzLJFO4gQ=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw=
+gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
+gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
+k8s.io/api v0.0.0-20190620084959-7cf5895f2711/go.mod h1:TBhBqb1AWbBQbW3XRusr7n7E4v2+5ZY8r8sAMnyFC5A=
+k8s.io/apimachinery v0.0.0-20190612205821-1799e75a0719/go.mod h1:I4A+glKBHiTgiEjQiCCQfCAIcIMFGt291SmsvcrFzJA=
+k8s.io/apimachinery v0.19.4/go.mod h1:DnPGDnARWFvYa3pMHgSxtbZb7gpzzAZ1pTfaUNDVlmA=
+k8s.io/client-go v0.0.0-20190620085101-78d2af792bab/go.mod h1:E95RaSlHr79aHaX0aGSwcPNfygDiPKOVXdmivCIZT0k=
+k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
+k8s.io/klog v0.3.1/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk=
+k8s.io/klog v1.0.0/go.mod h1:4Bi6QPql/J/LkTDqv7R/cd3hPo4k2DG6Ptcz060Ez5I=
+k8s.io/klog/v2 v2.0.0/go.mod h1:PBfzABfn139FHAV07az/IF9Wp1bkk3vpT2XSJ76fSDE=
+k8s.io/klog/v2 v2.2.0/go.mod h1:Od+F08eJP+W3HUb4pSrPpgp9DGU4GzlpG/TmITuYh/Y=
+k8s.io/kube-openapi v0.0.0-20190228160746-b3a7cee44a30/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc=
+k8s.io/kube-openapi v0.0.0-20200805222855-6aeccd4b50c6/go.mod h1:UuqjUnNftUyPE5H64/qeyjQoUZhGpeFDVdxjTeEVN2o=
+k8s.io/utils v0.0.0-20190221042446-c2654d5206da/go.mod h1:8k8uAuAQ0rXslZKaEWd0c3oVhZz7sSzSiPnVZayjIX0=
+rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
+sigs.k8s.io/structured-merge-diff/v4 v4.0.1/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK1F7G282QMXDPYydCw=
+sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o=
+sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc=

internal/oci/args.go

@@ -0,0 +1,115 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	specFileName = "config.json"
+)
+
+// GetBundleDir returns the bundle directory or default depending on the
+// supplied command line arguments.
+func GetBundleDir(args []string) (string, error) {
+	bundleDir, err := GetBundleDirFromArgs(args)
+	if err != nil {
+		return "", fmt.Errorf("error getting bundle dir from args: %v", err)
+	}
+
+	return bundleDir, nil
+}
+
+// GetBundleDirFromArgs checks the specified slice of strings (argv) for a 'bundle' flag as allowed by runc.
+// The following are supported:
+// --bundle{{SEP}}BUNDLE_PATH
+// -bundle{{SEP}}BUNDLE_PATH
+// -b{{SEP}}BUNDLE_PATH
+// where {{SEP}} is either ' ' or '='
+func GetBundleDirFromArgs(args []string) (string, error) {
+	var bundleDir string
+
+	for i := 0; i < len(args); i++ {
+		param := args[i]
+
+		parts := strings.SplitN(param, "=", 2)
+		if !IsBundleFlag(parts[0]) {
+			continue
+		}
+
+		// The flag has the format --bundle=/path
+		if len(parts) == 2 {
+			bundleDir = parts[1]
+			continue
+		}
+
+		// The flag has the format --bundle /path
+		if i+1 < len(args) {
+			bundleDir = args[i+1]
+			i++
+			continue
+		}
+
+		// --bundle / -b was the last element of args
+		return "", fmt.Errorf("bundle option requires an argument")
+	}
+
+	return bundleDir, nil
+}
+
+// GetSpecFilePath returns the expected path to the OCI specification file for the given
+// bundle directory.
+func GetSpecFilePath(bundleDir string) string {
+	specFilePath := filepath.Join(bundleDir, specFileName)
+	return specFilePath
+}
+
+// IsBundleFlag is a helper function that checks wither the specified argument represents
+// a bundle flag (--bundle or -b)
+func IsBundleFlag(arg string) bool {
+	if !strings.HasPrefix(arg, "-") {
+		return false
+	}
+
+	trimmed := strings.TrimLeft(arg, "-")
+	return trimmed == "b" || trimmed == "bundle"
+}
+
+// HasCreateSubcommand checks the supplied arguments for a 'create' subcommand
+func HasCreateSubcommand(args []string) bool {
+	var previousWasBundle bool
+	for _, a := range args {
+		// We check for '--bundle create' explicitly to ensure that we
+		// don't inadvertently trigger a modification if the bundle directory
+		// is specified as `create`
+		if !previousWasBundle && IsBundleFlag(a) {
+			previousWasBundle = true
+			continue
+		}
+
+		if !previousWasBundle && a == "create" {
+			return true
+		}
+
+		previousWasBundle = false
+	}
+
+	return false
+}

internal/oci/args_test.go

@@ -0,0 +1,184 @@
+package oci
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetBundleDir(t *testing.T) {
+	type expected struct {
+		bundle  string
+		isError bool
+	}
+	testCases := []struct {
+		argv     []string
+		expected expected
+	}{
+		{
+			argv: []string{},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"create"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"--bundle"},
+			expected: expected{
+				isError: true,
+			},
+		},
+		{
+			argv: []string{"-b"},
+			expected: expected{
+				isError: true,
+			},
+		},
+		{
+			argv: []string{"--bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"--not-bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"--"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"-bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"--bundle=/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b=/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b=/foo/=bar"},
+			expected: expected{
+				bundle: "/foo/=bar",
+			},
+		},
+		{
+			argv: []string{"-b", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"create", "-b", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b", "create", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+		{
+			argv: []string{"-b=create", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+		{
+			argv: []string{"-b", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		bundle, err := GetBundleDir(tc.argv)
+
+		if tc.expected.isError {
+			require.Errorf(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoErrorf(t, err, "%d: %v", i, tc)
+		}
+
+		require.Equalf(t, tc.expected.bundle, bundle, "%d: %v", i, tc)
+	}
+}
+
+func TestGetSpecFilePathAppendsFilename(t *testing.T) {
+	testCases := []struct {
+		bundleDir string
+		expected  string
+	}{
+		{
+			bundleDir: "",
+			expected:  "config.json",
+		},
+		{
+			bundleDir: "/not/empty/",
+			expected:  "/not/empty/config.json",
+		},
+		{
+			bundleDir: "not/absolute",
+			expected:  "not/absolute/config.json",
+		},
+	}
+
+	for i, tc := range testCases {
+		specPath := GetSpecFilePath(tc.bundleDir)
+
+		require.Equalf(t, tc.expected, specPath, "%d: %v", i, tc)
+	}
+}
+
+func TestHasCreateSubcommand(t *testing.T) {
+	testCases := []struct {
+		args         []string
+		shouldModify bool
+	}{
+		{
+			shouldModify: false,
+		},
+		{
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+		{
+			args:         []string{"--bundle=create"},
+			shouldModify: false,
+		},
+		{
+			args:         []string{"--bundle", "create"},
+			shouldModify: false,
+		},
+		{
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		require.Equal(t, tc.shouldModify, HasCreateSubcommand(tc.args), "%d: %v", i, tc)
+	}
+}

internal/oci/runtime.go

@@ -0,0 +1,25 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+//go:generate moq -stub -out runtime_mock.go . Runtime
+
+// Runtime is an interface for a runtime shim. The Exec method accepts a list
+// of command line arguments, and returns an error / nil.
+type Runtime interface {
+	Exec([]string) error
+}

internal/oci/runtime_low_level.go

@@ -0,0 +1,61 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os/exec"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// NewLowLevelRuntime creates a Runtime that wraps a low-level runtime executable.
+// The executable specified is taken from the list of supplied candidates, with the first match
+// present in the PATH being selected.
+func NewLowLevelRuntime(candidates ...string) (Runtime, error) {
+	return NewLowLevelRuntimeWithLogger(log.StandardLogger(), candidates...)
+}
+
+// NewLowLevelRuntimeWithLogger creates a Runtime as with NewLowLevelRuntime using the specified logger.
+func NewLowLevelRuntimeWithLogger(logger *log.Logger, candidates ...string) (Runtime, error) {
+	runtimePath, err := findRuntime(logger, candidates)
+	if err != nil {
+		return nil, fmt.Errorf("error locating runtime: %v", err)
+	}
+
+	return NewRuntimeForPathWithLogger(logger, runtimePath)
+}
+
+// findRuntime checks elements in a list of supplied candidates for a matching executable in the PATH.
+// The absolute path to the first match is returned.
+func findRuntime(logger *log.Logger, candidates []string) (string, error) {
+	if len(candidates) == 0 {
+		return "", fmt.Errorf("at least one runtime candidate must be specified")
+	}
+
+	for _, candidate := range candidates {
+		logger.Infof("Looking for runtime binary '%v'", candidate)
+		runcPath, err := exec.LookPath(candidate)
+		if err == nil {
+			logger.Infof("Found runtime binary '%v'", runcPath)
+			return runcPath, nil
+		}
+		logger.Warnf("Runtime binary '%v' not found: %v", candidate, err)
+	}
+
+	return "", fmt.Errorf("no runtime binary found from candidate list: %v", candidates)
+}

internal/oci/runtime_mock.go

@@ -0,0 +1,76 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package oci
+
+import (
+	"sync"
+)
+
+// Ensure, that RuntimeMock does implement Runtime.
+// If this is not the case, regenerate this file with moq.
+var _ Runtime = &RuntimeMock{}
+
+// RuntimeMock is a mock implementation of Runtime.
+//
+// 	func TestSomethingThatUsesRuntime(t *testing.T) {
+//
+// 		// make and configure a mocked Runtime
+// 		mockedRuntime := &RuntimeMock{
+// 			ExecFunc: func(strings []string) error {
+// 				panic("mock out the Exec method")
+// 			},
+// 		}
+//
+// 		// use mockedRuntime in code that requires Runtime
+// 		// and then make assertions.
+//
+// 	}
+type RuntimeMock struct {
+	// ExecFunc mocks the Exec method.
+	ExecFunc func(strings []string) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Exec holds details about calls to the Exec method.
+		Exec []struct {
+			// Strings is the strings argument value.
+			Strings []string
+		}
+	}
+	lockExec sync.RWMutex
+}
+
+// Exec calls ExecFunc.
+func (mock *RuntimeMock) Exec(strings []string) error {
+	callInfo := struct {
+		Strings []string
+	}{
+		Strings: strings,
+	}
+	mock.lockExec.Lock()
+	mock.calls.Exec = append(mock.calls.Exec, callInfo)
+	mock.lockExec.Unlock()
+	if mock.ExecFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.ExecFunc(strings)
+}
+
+// ExecCalls gets all the calls that were made to Exec.
+// Check the length with:
+//     len(mockedRuntime.ExecCalls())
+func (mock *RuntimeMock) ExecCalls() []struct {
+	Strings []string
+} {
+	var calls []struct {
+		Strings []string
+	}
+	mock.lockExec.RLock()
+	calls = mock.calls.Exec
+	mock.lockExec.RUnlock()
+	return calls
+}

internal/oci/runtime_path.go

@@ -0,0 +1,70 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// pathRuntime wraps the path that a binary and defines the semanitcs for how to exec into it.
+// This can be used to wrap an OCI-compliant low-level runtime binary, allowing it to be used through the
+// Runtime internface.
+type pathRuntime struct {
+	logger      *log.Logger
+	path        string
+	execRuntime Runtime
+}
+
+var _ Runtime = (*pathRuntime)(nil)
+
+// NewRuntimeForPath creates a Runtime for the specified path with the standard logger
+func NewRuntimeForPath(path string) (Runtime, error) {
+	return NewRuntimeForPathWithLogger(log.StandardLogger(), path)
+}
+
+// NewRuntimeForPathWithLogger creates a Runtime for the specified logger and path
+func NewRuntimeForPathWithLogger(logger *log.Logger, path string) (Runtime, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return nil, fmt.Errorf("invalid path '%v': %v", path, err)
+	}
+	if info.IsDir() || info.Mode()&0111 == 0 {
+		return nil, fmt.Errorf("specified path '%v' is not an executable file", path)
+	}
+
+	shim := pathRuntime{
+		logger:      logger,
+		path:        path,
+		execRuntime: syscallExec{},
+	}
+
+	return &shim, nil
+}
+
+// Exec exces into the binary at the path from the pathRuntime struct, passing it the supplied arguments
+// after ensuring that the first argument is the path of the target binary.
+func (s pathRuntime) Exec(args []string) error {
+	runtimeArgs := []string{s.path}
+	if len(args) > 1 {
+		runtimeArgs = append(runtimeArgs, args[1:]...)
+	}
+
+	return s.execRuntime.Exec(runtimeArgs)
+}

internal/oci/runtime_path_test.go

@@ -0,0 +1,97 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+package oci
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPathRuntimeConstructor(t *testing.T) {
+	r, err := NewRuntimeForPath("////an/invalid/path")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath("/tmp")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath("/dev/null")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath("/bin/sh")
+	require.NoError(t, err)
+
+	f, ok := r.(*pathRuntime)
+	require.True(t, ok)
+
+	require.Equal(t, "/bin/sh", f.path)
+}
+
+func TestPathRuntimeForwardsArgs(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		execRuntimeError error
+		args             []string
+	}{
+		{},
+		{
+			args: []string{"shouldBeReplaced"},
+		},
+		{
+			args: []string{"shouldBeReplaced", "arg1"},
+		},
+		{
+			execRuntimeError: fmt.Errorf("exec error"),
+		},
+	}
+
+	for _, tc := range testCases {
+		mockedRuntime := &RuntimeMock{
+			ExecFunc: func(strings []string) error {
+				return tc.execRuntimeError
+			},
+		}
+		r := pathRuntime{
+			logger:      logger,
+			path:        "runtime",
+			execRuntime: mockedRuntime,
+		}
+		err := r.Exec(tc.args)
+
+		require.ErrorIs(t, err, tc.execRuntimeError)
+
+		calls := mockedRuntime.ExecCalls()
+		require.Len(t, calls, 1)
+
+		numArgs := len(tc.args)
+		if numArgs == 0 {
+			numArgs = 1
+		}
+
+		require.Len(t, calls[0].Strings, numArgs)
+		require.Equal(t, "runtime", calls[0].Strings[0])
+
+		if numArgs > 1 {
+			require.EqualValues(t, tc.args[1:], calls[0].Strings[1:])
+		}
+	}
+}

internal/oci/runtime_syscall_exec.go

@@ -0,0 +1,38 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+type syscallExec struct{}
+
+var _ Runtime = (*syscallExec)(nil)
+
+func (r syscallExec) Exec(args []string) error {
+	err := syscall.Exec(args[0], args, os.Environ())
+	if err != nil {
+		return fmt.Errorf("could not exec '%v': %v", args[0], err)
+	}
+
+	// syscall.Exec is not expected to return. This is an error state regardless of whether
+	// err is nil or not.
+	return fmt.Errorf("unexpected return from exec '%v'", args[0])
+}

internal/oci/spec.go

@@ -0,0 +1,35 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+// SpecModifier is a function that accepts a pointer to an OCI Srec and returns an
+// error. The intention is that the function would modify the spec in-place.
+type SpecModifier func(*oci.Spec) error
+
+//go:generate moq -stub -out spec_mock.go . Spec
+
+// Spec defines the operations to be performed on an OCI specification
+type Spec interface {
+	Load() error
+	Flush() error
+	Modify(SpecModifier) error
+	LookupEnv(string) (string, bool)
+}

internal/oci/spec_file.go

@@ -0,0 +1,153 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+type fileSpec struct {
+	*oci.Spec
+	path string
+}
+
+var _ Spec = (*fileSpec)(nil)
+
+// NewSpecFromArgs creates fileSpec based on the command line arguments passed to the
+// application
+func NewSpecFromArgs(args []string) (Spec, string, error) {
+	bundleDir, err := GetBundleDir(args)
+	if err != nil {
+		return nil, "", fmt.Errorf("error getting bundle directory: %v", err)
+	}
+
+	ociSpecPath := GetSpecFilePath(bundleDir)
+
+	ociSpec := NewSpecFromFile(ociSpecPath)
+
+	return ociSpec, bundleDir, nil
+}
+
+// NewSpecFromFile creates an object that encapsulates a file-backed OCI spec.
+// This can be used to read from the file, modify the spec, and write to the
+// same file.
+func NewSpecFromFile(filepath string) Spec {
+	oci := fileSpec{
+		path: filepath,
+	}
+
+	return &oci
+}
+
+// Load reads the contents of an OCI spec from file to be referenced internally.
+// The file is opened "read-only"
+func (s *fileSpec) Load() error {
+	specFile, err := os.Open(s.path)
+	if err != nil {
+		return fmt.Errorf("error opening OCI specification file: %v", err)
+	}
+	defer specFile.Close()
+
+	return s.loadFrom(specFile)
+}
+
+// loadFrom reads the contents of the OCI spec from the specified io.Reader.
+func (s *fileSpec) loadFrom(reader io.Reader) error {
+	decoder := json.NewDecoder(reader)
+
+	var spec oci.Spec
+	err := decoder.Decode(&spec)
+	if err != nil {
+		return fmt.Errorf("error reading OCI specification: %v", err)
+	}
+
+	s.Spec = &spec
+	return nil
+}
+
+// Modify applies the specified SpecModifier to the stored OCI specification.
+func (s *fileSpec) Modify(f SpecModifier) error {
+	if s.Spec == nil {
+		return fmt.Errorf("no spec loaded for modification")
+	}
+	return f(s.Spec)
+}
+
+// Flush writes the stored OCI specification to the filepath specifed by the path member.
+// The file is truncated upon opening, overwriting any existing contents.
+func (s fileSpec) Flush() error {
+	if s.Spec == nil {
+		return fmt.Errorf("no OCI specification loaded")
+	}
+
+	specFile, err := os.Create(s.path)
+	if err != nil {
+		return fmt.Errorf("error opening OCI specification file: %v", err)
+	}
+	defer specFile.Close()
+
+	return s.flushTo(specFile)
+}
+
+// flushTo writes the stored OCI specification to the specified io.Writer.
+func (s fileSpec) flushTo(writer io.Writer) error {
+	if s.Spec == nil {
+		return nil
+	}
+	encoder := json.NewEncoder(writer)
+
+	err := encoder.Encode(s.Spec)
+	if err != nil {
+		return fmt.Errorf("error writing OCI specification: %v", err)
+	}
+
+	return nil
+}
+
+// LookupEnv mirrors os.LookupEnv for the OCI specification. It
+// retrieves the value of the environment variable named
+// by the key. If the variable is present in the environment the
+// value (which may be empty) is returned and the boolean is true.
+// Otherwise the returned value will be empty and the boolean will
+// be false.
+func (s fileSpec) LookupEnv(key string) (string, bool) {
+	if s.Spec == nil || s.Spec.Process == nil {
+		return "", false
+	}
+
+	for _, env := range s.Spec.Process.Env {
+		if !strings.HasPrefix(env, key) {
+			continue
+		}
+
+		parts := strings.SplitN(env, "=", 2)
+		if parts[0] == key {
+			if len(parts) < 2 {
+				return "", true
+			}
+			return parts[1], true
+		}
+	}
+
+	return "", false
+}

internal/oci/spec_file_test.go

@@ -0,0 +1,252 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLookupEnv(t *testing.T) {
+	const envName = "TEST_ENV"
+	testCases := []struct {
+		spec          *specs.Spec
+		expectedValue string
+		expectedExits bool
+	}{
+		{
+			// nil spec
+			spec:          nil,
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// nil process
+			spec:          &specs.Spec{},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// nil env
+			spec: &specs.Spec{
+				Process: &specs.Process{},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// empty env
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// different env set
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"SOMETHING_ELSE=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// same prefix
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV_BUT_NOT=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// same suffix
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"NOT_TEST_ENV=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// set blank
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV="}},
+			},
+			expectedValue: "",
+			expectedExits: true,
+		},
+		{
+			// set no-equals
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV"}},
+			},
+			expectedValue: "",
+			expectedExits: true,
+		},
+		{
+			// set value
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV=something"}},
+			},
+			expectedValue: "something",
+			expectedExits: true,
+		},
+		{
+			// set with equals
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV=something=somethingelse"}},
+			},
+			expectedValue: "something=somethingelse",
+			expectedExits: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		spec := fileSpec{
+			Spec: tc.spec,
+		}
+
+		value, exists := spec.LookupEnv(envName)
+
+		require.Equal(t, tc.expectedValue, value, "%d: %v", i, tc)
+		require.Equal(t, tc.expectedExits, exists, "%d: %v", i, tc)
+	}
+}
+
+func TestLoadFrom(t *testing.T) {
+	testCases := []struct {
+		contents []byte
+		isError  bool
+		spec     *specs.Spec
+	}{
+		{
+			contents: []byte{},
+			isError:  true,
+		},
+		{
+			contents: []byte("{}"),
+			isError:  false,
+			spec:     &specs.Spec{},
+		},
+	}
+
+	for i, tc := range testCases {
+		spec := fileSpec{}
+		err := spec.loadFrom(bytes.NewReader(tc.contents))
+
+		if tc.isError {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+
+		if tc.spec == nil {
+			require.Nil(t, spec.Spec, "%d: %v", i, tc)
+		} else {
+			require.EqualValues(t, tc.spec, spec.Spec, "%d: %v", i, tc)
+		}
+	}
+}
+
+func TestFlushTo(t *testing.T) {
+	testCases := []struct {
+		isError  bool
+		spec     *specs.Spec
+		contents string
+	}{
+		{
+			spec: nil,
+		},
+		{
+			spec:     &specs.Spec{},
+			contents: "{\"ociVersion\":\"\"}\n",
+		},
+	}
+
+	for i, tc := range testCases {
+		buffer := bytes.Buffer{}
+
+		spec := fileSpec{Spec: tc.spec}
+		err := spec.flushTo(&buffer)
+
+		if tc.isError {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+
+		require.EqualValues(t, tc.contents, buffer.String(), "%d: %v", i, tc)
+	}
+
+	// Add a simple test for a writer that returns an error when writing
+	spec := fileSpec{Spec: &specs.Spec{}}
+	err := spec.flushTo(errorWriter{})
+	require.Error(t, err)
+}
+
+func TestModify(t *testing.T) {
+
+	testCases := []struct {
+		spec          *specs.Spec
+		modifierError error
+	}{
+		{
+			spec: nil,
+		},
+		{
+			spec: &specs.Spec{},
+		},
+		{
+			spec:          &specs.Spec{},
+			modifierError: fmt.Errorf("error in modifier"),
+		},
+	}
+
+	for i, tc := range testCases {
+		spec := fileSpec{Spec: tc.spec}
+
+		modifier := func(spec *specs.Spec) error {
+			if tc.modifierError == nil {
+				spec.Version = "updated"
+			}
+			return tc.modifierError
+		}
+
+		err := spec.Modify(modifier)
+
+		if tc.spec == nil {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else if tc.modifierError != nil {
+			require.EqualError(t, err, tc.modifierError.Error(), "%d: %v", i, tc)
+			require.EqualValues(t, &specs.Spec{}, spec.Spec, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+			require.Equal(t, "updated", spec.Spec.Version, "%d: %v", i, tc)
+		}
+	}
+}
+
+// errorWriter implements the io.Writer interface, always returning an error when
+// writing.
+type errorWriter struct{}
+
+func (e errorWriter) Write([]byte) (int, error) {
+	return 0, fmt.Errorf("error writing")
+}

internal/oci/spec_mock.go

@@ -0,0 +1,201 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package oci
+
+import (
+	"sync"
+)
+
+// Ensure, that SpecMock does implement Spec.
+// If this is not the case, regenerate this file with moq.
+var _ Spec = &SpecMock{}
+
+// SpecMock is a mock implementation of Spec.
+//
+// 	func TestSomethingThatUsesSpec(t *testing.T) {
+//
+// 		// make and configure a mocked Spec
+// 		mockedSpec := &SpecMock{
+// 			FlushFunc: func() error {
+// 				panic("mock out the Flush method")
+// 			},
+// 			LoadFunc: func() error {
+// 				panic("mock out the Load method")
+// 			},
+// 			LookupEnvFunc: func(s string) (string, bool) {
+// 				panic("mock out the LookupEnv method")
+// 			},
+// 			ModifyFunc: func(specModifier SpecModifier) error {
+// 				panic("mock out the Modify method")
+// 			},
+// 		}
+//
+// 		// use mockedSpec in code that requires Spec
+// 		// and then make assertions.
+//
+// 	}
+type SpecMock struct {
+	// FlushFunc mocks the Flush method.
+	FlushFunc func() error
+
+	// LoadFunc mocks the Load method.
+	LoadFunc func() error
+
+	// LookupEnvFunc mocks the LookupEnv method.
+	LookupEnvFunc func(s string) (string, bool)
+
+	// ModifyFunc mocks the Modify method.
+	ModifyFunc func(specModifier SpecModifier) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Flush holds details about calls to the Flush method.
+		Flush []struct {
+		}
+		// Load holds details about calls to the Load method.
+		Load []struct {
+		}
+		// LookupEnv holds details about calls to the LookupEnv method.
+		LookupEnv []struct {
+			// S is the s argument value.
+			S string
+		}
+		// Modify holds details about calls to the Modify method.
+		Modify []struct {
+			// SpecModifier is the specModifier argument value.
+			SpecModifier SpecModifier
+		}
+	}
+	lockFlush     sync.RWMutex
+	lockLoad      sync.RWMutex
+	lockLookupEnv sync.RWMutex
+	lockModify    sync.RWMutex
+}
+
+// Flush calls FlushFunc.
+func (mock *SpecMock) Flush() error {
+	callInfo := struct {
+	}{}
+	mock.lockFlush.Lock()
+	mock.calls.Flush = append(mock.calls.Flush, callInfo)
+	mock.lockFlush.Unlock()
+	if mock.FlushFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.FlushFunc()
+}
+
+// FlushCalls gets all the calls that were made to Flush.
+// Check the length with:
+//     len(mockedSpec.FlushCalls())
+func (mock *SpecMock) FlushCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockFlush.RLock()
+	calls = mock.calls.Flush
+	mock.lockFlush.RUnlock()
+	return calls
+}
+
+// Load calls LoadFunc.
+func (mock *SpecMock) Load() error {
+	callInfo := struct {
+	}{}
+	mock.lockLoad.Lock()
+	mock.calls.Load = append(mock.calls.Load, callInfo)
+	mock.lockLoad.Unlock()
+	if mock.LoadFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.LoadFunc()
+}
+
+// LoadCalls gets all the calls that were made to Load.
+// Check the length with:
+//     len(mockedSpec.LoadCalls())
+func (mock *SpecMock) LoadCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockLoad.RLock()
+	calls = mock.calls.Load
+	mock.lockLoad.RUnlock()
+	return calls
+}
+
+// LookupEnv calls LookupEnvFunc.
+func (mock *SpecMock) LookupEnv(s string) (string, bool) {
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockLookupEnv.Lock()
+	mock.calls.LookupEnv = append(mock.calls.LookupEnv, callInfo)
+	mock.lockLookupEnv.Unlock()
+	if mock.LookupEnvFunc == nil {
+		var (
+			sOut string
+			bOut bool
+		)
+		return sOut, bOut
+	}
+	return mock.LookupEnvFunc(s)
+}
+
+// LookupEnvCalls gets all the calls that were made to LookupEnv.
+// Check the length with:
+//     len(mockedSpec.LookupEnvCalls())
+func (mock *SpecMock) LookupEnvCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockLookupEnv.RLock()
+	calls = mock.calls.LookupEnv
+	mock.lockLookupEnv.RUnlock()
+	return calls
+}
+
+// Modify calls ModifyFunc.
+func (mock *SpecMock) Modify(specModifier SpecModifier) error {
+	callInfo := struct {
+		SpecModifier SpecModifier
+	}{
+		SpecModifier: specModifier,
+	}
+	mock.lockModify.Lock()
+	mock.calls.Modify = append(mock.calls.Modify, callInfo)
+	mock.lockModify.Unlock()
+	if mock.ModifyFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.ModifyFunc(specModifier)
+}
+
+// ModifyCalls gets all the calls that were made to Modify.
+// Check the length with:
+//     len(mockedSpec.ModifyCalls())
+func (mock *SpecMock) ModifyCalls() []struct {
+	SpecModifier SpecModifier
+} {
+	var calls []struct {
+		SpecModifier SpecModifier
+	}
+	mock.lockModify.RLock()
+	calls = mock.calls.Modify
+	mock.lockModify.RUnlock()
+	return calls
+}

internal/oci/spec_test.go

@@ -0,0 +1,58 @@
+package oci
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMaintainSpec(t *testing.T) {
+	moduleRoot, err := getModuleRoot()
+	require.NoError(t, err)
+
+	files := []string{
+		"config.clone3.json",
+	}
+
+	for _, f := range files {
+		inputSpecPath := filepath.Join(moduleRoot, "test/input", f)
+
+		spec := NewSpecFromFile(inputSpecPath).(*fileSpec)
+
+		spec.Load()
+
+		outputSpecPath := filepath.Join(moduleRoot, "test/output", f)
+		spec.path = outputSpecPath
+		spec.Flush()
+
+		inputContents, err := os.ReadFile(inputSpecPath)
+		require.NoError(t, err)
+
+		outputContents, err := os.ReadFile(outputSpecPath)
+		require.NoError(t, err)
+
+		require.JSONEq(t, string(inputContents), string(outputContents))
+	}
+}
+
+func getModuleRoot() (string, error) {
+	_, filename, _, _ := runtime.Caller(0)
+
+	return hasGoMod(filename)
+}
+
+func hasGoMod(dir string) (string, error) {
+	if dir == "" || dir == "/" {
+		return "", fmt.Errorf("module root not found")
+	}
+
+	_, err := os.Stat(filepath.Join(dir, "go.mod"))
+	if err != nil {
+		return hasGoMod(filepath.Dir(dir))
+	}
+	return dir, nil
+}

packaging/debian/changelog

@@ -1,4 +1,78 @@
-nvidia-container-toolkit (1.3.0-1)  UNRELEASED; urgency=medium
+nvidia-container-toolkit (1.7.0~rc.1-1) experimental; urgency=medium
+
+  * Specify containerd runtime type as string in config tools to remove dependency on containerd package
+  * Add supported-driver-capabilities config option to allow for a subset of all driver capabilities to be specified
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Thu, 25 Nov 2021 11:36:29 +0100
+
+nvidia-container-toolkit (1.6.0-1) UNRELEASED; urgency=medium
+
+  * Promote 1.6.0~rc.3-1 to 1.6.0-1
+  * Fix unnecessary logging to stderr instead of configured nvidia-container-runtime log file
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Wed, 17 Nov 2021 09:25:15 +0100
+
+nvidia-container-toolkit (1.6.0~rc.3-1) experimental; urgency=medium
+
+  * Add supported-driver-capabilities config option to the nvidia-container-toolkit
+  * Move OCI and command line checks for runtime to internal oci package
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Mon, 15 Nov 2021 13:02:23 +0100
+
+nvidia-container-toolkit (1.6.0~rc.2-1) experimental; urgency=medium
+
+  * Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Fri, 05 Nov 2021 12:24:05 +0200
+
+nvidia-container-toolkit (1.6.0~rc.1-1) experimental; urgency=medium
+
+  * Add AARCH64 package for Amazon Linux 2
+  * Include nvidia-container-runtime into nvidia-container-toolkit package
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Mon, 06 Sep 2021 12:24:05 +0200
+
+nvidia-container-toolkit (1.5.1-1) UNRELEASED; urgency=medium
+
+  * Fix bug where Docker Swarm device selection is ignored if
+    NVIDIA_VISIBLE_DEVICES is also set
+  * Improve unit testing by using require package and adding coverage reports
+  * Remove unneeded go dependencies by running go mod tidy
+  * Move contents of pkg directory to cmd for CLI tools
+  * Ensure make binary target explicitly sets GOOS
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Mon, 14 Jun 2021 09:00:00 -0700
+
+nvidia-container-toolkit (1.5.0-1) UNRELEASED; urgency=medium
+
+  * Add dependence on libnvidia-container-tools >= 1.4.0
+  * Add golang check targets to Makefile
+  * Add Jenkinsfile definition for build targets
+  * Move docker.mk to docker folder
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Thu, 29 Apr 2021 03:12:43 -0700
+
+nvidia-container-toolkit (1.4.2-1) UNRELEASED; urgency=medium
+
+  * Add dependence on libnvidia-container-tools >= 1.3.3
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Fri, 05 Feb 2021 02:24:36 -0700
+
+nvidia-container-toolkit (1.4.1-1) UNRELEASED; urgency=medium
+
+  * Ignore NVIDIA_VISIBLE_DEVICES for containers with insufficent privileges
+  * Add dependence on libnvidia-container-tools >= 1.3.2
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Mon, 25 Jan 2021 02:18:04 -0700
+
+nvidia-container-toolkit (1.4.0-1) UNRELEASED; urgency=medium
+
+  * Add 'compute' capability to list of defaults
+  * Add dependence on libnvidia-container-tools >= 1.3.1
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Fri, 11 Dec 2020 18:29:23 -0700
+
+nvidia-container-toolkit (1.3.0-1) UNRELEASED; urgency=medium
 
   * Promote 1.3.0~rc.2-1 to 1.3.0-1
   * Add dependence on libnvidia-container-tools >= 1.3.0
@@ -56,7 +130,7 @@ nvidia-container-toolkit (1.1.0-1) UNRELEASED; urgency=medium
 
   * 4e4de762 Update build system to support multi-arch builds
   * fcc1d116 Add support for MIG (Multi-Instance GPUs)
-  * d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_* 
+  * d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*
   * 60f165ad Add no-pivot option to toolkit
 
  -- NVIDIA CORPORATION <cudatools@nvidia.com>  Fri, 15 May 2020 12:05:32 -0700

packaging/debian/control

@@ -10,8 +10,8 @@ Build-Depends: debhelper (>= 9)
 
 Package: nvidia-container-toolkit
 Architecture: any
-Depends: ${misc:Depends}, libnvidia-container-tools (>= 1.3.0), libnvidia-container-tools (<< 2.0.0)
-Breaks: nvidia-container-runtime (<< 2.0.0), nvidia-container-runtime-hook
-Replaces: nvidia-container-runtime (<< 2.0.0), nvidia-container-runtime-hook
+Depends: ${misc:Depends}, libnvidia-container-tools (>= @LIBNVIDIA_CONTAINER_VERSION@), libnvidia-container-tools (<< 2.0.0), libseccomp2
+Breaks: nvidia-container-runtime (<= 3.5.0-1), nvidia-container-runtime-hook
+Replaces: nvidia-container-runtime (<= 3.5.0-1), nvidia-container-runtime-hook
 Description: NVIDIA container runtime hook
  Provides a OCI hook to enable GPU support in containers.

packaging/debian/nvidia-container-toolkit.install

@@ -1,2 +1,3 @@
 config.toml /etc/nvidia-container-runtime
 nvidia-container-toolkit /usr/bin
+nvidia-container-runtime /usr/bin

packaging/debian/prepare

@@ -3,6 +3,7 @@
 set -e
 
 sed -i "s;@SECTION@;${SECTION:+$SECTION/};g" debian/control
+sed -i "s;@LIBNVIDIA_CONTAINER_VERSION@;${LIBNVIDIA_CONTAINER_VERSION:+$LIBNVIDIA_CONTAINER_VERSION};g" debian/control
 
 if [ -n "$DISTRIB" ]; then
     sed -i "s;UNRELEASED;$DISTRIB;" debian/changelog

packaging/rpm/SPECS/nvidia-container-toolkit.spec

@@ -11,24 +11,34 @@ URL: https://github.com/NVIDIA/nvidia-container-runtime
 License: Apache-2.0
 
 Source0: nvidia-container-toolkit
-Source1: config.toml
-Source2: oci-nvidia-hook
-Source3: oci-nvidia-hook.json
-Source4: LICENSE
+Source1: nvidia-container-runtime
+Source2: config.toml
+Source3: oci-nvidia-hook
+Source4: oci-nvidia-hook.json
+Source5: LICENSE
 
-Obsoletes: nvidia-container-runtime < 2.0.0, nvidia-container-runtime-hook
+Obsoletes: nvidia-container-runtime <= 3.5.0-1, nvidia-container-runtime-hook
+Provides: nvidia-container-runtime
 Provides: nvidia-container-runtime-hook
-Requires: libnvidia-container-tools >= 1.3.0, libnvidia-container-tools < 2.0.0
+Requires: libnvidia-container-tools >= %{libnvidia_container_version}, libnvidia-container-tools < 2.0.0
+
+%if 0%{?suse_version}
+Requires: libseccomp2
+Requires: libapparmor1
+%else
+Requires: libseccomp
+%endif
 
 %description
 Provides a OCI hook to enable GPU support in containers.
 
 %prep
-cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} .
+cp %{SOURCE0} %{SOURCE1} %{SOURCE2} %{SOURCE3} %{SOURCE4} %{SOURCE5} .
 
 %install
 mkdir -p %{buildroot}%{_bindir}
 install -m 755 -t %{buildroot}%{_bindir} nvidia-container-toolkit
+install -m 755 -t %{buildroot}%{_bindir} nvidia-container-runtime
 
 mkdir -p %{buildroot}/etc/nvidia-container-runtime
 install -m 644 -t %{buildroot}/etc/nvidia-container-runtime config.toml
@@ -48,11 +58,59 @@ rm -f %{_bindir}/nvidia-container-runtime-hook
 %files
 %license LICENSE
 %{_bindir}/nvidia-container-toolkit
+%{_bindir}/nvidia-container-runtime
 %config /etc/nvidia-container-runtime/config.toml
 /usr/libexec/oci/hooks.d/oci-nvidia-hook
 /usr/share/containers/oci/hooks.d/oci-nvidia-hook.json
 
 %changelog
+* Thu Nov 25 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.7.0-0.1.rc.1
+- Specify containerd runtime type as string in config tools to remove dependency on containerd package
+- Add supported-driver-capabilities config option to allow for a subset of all driver capabilities to be specified
+
+* Wed Nov 17 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.6.0-1
+- Promote 1.6.0-0.1.rc.3 to 1.6.0-1
+- Fix unnecessary logging to stderr instead of configured nvidia-container-runtime log file
+
+* Mon Nov 15 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.6.0-0.1.rc.3
+
+- Add supported-driver-capabilities config option to the nvidia-container-toolkit
+- Move OCI and command line checks for runtime to internal oci package
+
+* Fri Nov 05 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.6.0-0.1.rc.2
+
+- Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
+
+* Mon Sep 06 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.6.0-0.1.rc.1
+
+- Add AARCH64 package for Amazon Linux 2
+- Include nvidia-container-runtime into nvidia-container-toolkit package
+
+* Mon Jun 14 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.5.1-1
+
+- Fix bug where Docker Swarm device selection is ignored if NVIDIA_VISIBLE_DEVICES is also set
+- Improve unit testing by using require package and adding coverage reports
+- Remove unneeded go dependencies by running go mod tidy
+- Move contents of pkg directory to cmd for CLI tools
+- Ensure make binary target explicitly sets GOOS
+
+* Thu Apr 29 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.5.0-1
+- Add dependence on libnvidia-container-tools >= 1.4.0
+- Add golang check targets to Makefile
+- Add Jenkinsfile definition for build targets
+- Move docker.mk to docker folder
+
+* Fri Feb 05 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.4.2-1
+- Add dependence on libnvidia-container-tools >= 1.3.3
+
+* Mon Jan 25 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.4.1-1
+- Ignore NVIDIA_VISIBLE_DEVICES for containers with insufficent privileges
+- Add dependence on libnvidia-container-tools >= 1.3.2
+
+* Fri Dec 11 2020 NVIDIA CORPORATION <cudatools@nvidia.com> 1.4.0-1
+- Add 'compute' capability to list of defaults
+- Add dependence on libnvidia-container-tools >= 1.3.1
+
 * Wed Sep 16 2020 NVIDIA CORPORATION <cudatools@nvidia.com> 1.3.0-1
 - Promote 1.3.0-0.1.rc.2 to 1.3.0-1
 - Add dependence on libnvidia-container-tools >= 1.3.0
@@ -89,5 +147,5 @@ rm -f %{_bindir}/nvidia-container-runtime-hook
 * Fri May 15 2020 NVIDIA CORPORATION <cudatools@nvidia.com> 1.1.0-1
 - 4e4de762 Update build system to support multi-arch builds
 - fcc1d116 Add support for MIG (Multi-Instance GPUs)
-- d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_* 
+- d4ff0416 Add ability to merge envars of the form NVIDIA_VISIBLE_DEVICES_*
 - 60f165ad Add no-pivot option to toolkit

pkg/capabilities.go

@@ -1,27 +0,0 @@
-package main
-
-import (
-	"log"
-)
-
-func capabilityToCLI(cap string) string {
-	switch cap {
-	case "compute":
-		return "--compute"
-	case "compat32":
-		return "--compat32"
-	case "graphics":
-		return "--graphics"
-	case "utility":
-		return "--utility"
-	case "video":
-		return "--video"
-	case "display":
-		return "--display"
-	case "ngx":
-		return "--ngx"
-	default:
-		log.Panicln("unknown driver capability:", cap)
-	}
-	return ""
-}

scripts/build-all-components.sh

@@ -0,0 +1,76 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to build the packages for the components of the NVIDIA
+# Container Stack. These include the nvidia-container-toolkit in this repository
+# as well as the components included in the third_party folder.
+# All required packages are generated in the specified dist folder.
+
+function assert_usage() {
+    echo "Missing argument $1"
+    echo "$(basename ${BASH_SOURCE[0]}) TARGET"
+    exit 1
+}
+
+set -e -x
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
+
+if [[ $# -ne 1 ]]; then
+    assert_usage "TARGET"
+fi
+
+TARGET=$1
+
+: ${DIST_DIR:=${PROJECT_ROOT}/dist}
+export DIST_DIR
+
+echo "Building ${TARGET} for all packages to ${DIST_DIR}"
+
+: ${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}
+: ${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}
+: ${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}
+: ${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}
+
+
+${SCRIPTS_DIR}/get-component-versions.sh
+
+# Build libnvidia-container
+make -C ${LIBNVIDIA_CONTAINER_ROOT} -f mk/docker.mk ${TARGET}
+
+# Build nvidia-container-toolkit
+make -C ${NVIDIA_CONTAINER_TOOLKIT_ROOT} ${TARGET}
+
+if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_VERSION} ]]; then
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+fi
+
+# We set the TOOLKIT_VERSION, TOOLKIT_TAG for the nvidia-container-runtime and nvidia-docker targets
+# The LIB_TAG is also overridden to match the TOOLKIT_TAG.
+# Build nvidia-container-runtime
+make -C ${NVIDIA_CONTAINER_RUNTIME_ROOT} \
+    LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+    TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
+    TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+        ${TARGET}
+
+# Build nvidia-docker2
+make -C ${NVIDIA_DOCKER_ROOT} \
+    LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+    TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
+    TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+        ${TARGET}

scripts/get-component-versions.sh

@@ -0,0 +1,65 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to build the packages for the components of the NVIDIA
+# Container Stack. These include the nvidia-container-toolkit in this repository
+# as well as the components included in the third_party folder.
+# All required packages are generated in the specified dist folder.
+
+function assert_usage() {
+    exit 1
+}
+
+set -e
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
+
+: ${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}
+: ${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}
+: ${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}
+: ${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}
+
+# Get version for libnvidia-container
+libnvidia_container_version_tag=$(grep "#define NVC_VERSION" ${LIBNVIDIA_CONTAINER_ROOT}/src/nvc.h \
+    | sed -e 's/#define NVC_VERSION[[:space:]]"\(.*\)"/\1/')
+
+# Get version for nvidia-container-toolit
+nvidia_container_toolkit_version=$(grep -m 1 "^LIB_VERSION := " ${NVIDIA_CONTAINER_TOOLKIT_ROOT}/Makefile | sed -e 's/LIB_VERSION :=[[:space:]]\(.*\)[[:space:]]*/\1/')
+nvidia_container_toolkit_tag=$(grep -m 1 "^LIB_TAG .= " ${NVIDIA_CONTAINER_TOOLKIT_ROOT}/Makefile | sed -e 's/LIB_TAG .=[[:space:]]\(.*\)[[:space:]]*/\1/')
+nvidia_container_toolkit_version_tag="${nvidia_container_toolkit_version}${nvidia_container_toolkit_tag:+~${nvidia_container_toolkit_tag}}"
+
+# Get version for nvidia-container-runtime
+nvidia_container_runtime_version=$(grep -m 1 "^LIB_VERSION := " ${NVIDIA_CONTAINER_RUNTIME_ROOT}/Makefile | sed -e 's/LIB_VERSION :=[[:space:]]\(.*\)[[:space:]]*/\1/')
+nvidia_container_runtime_tag=$(grep -m 1 "^LIB_TAG .= " ${NVIDIA_CONTAINER_RUNTIME_ROOT}/Makefile | sed -e 's/LIB_TAG .=[[:space:]]\(.*\)[[:space:]]*/\1/')
+nvidia_container_runtime_version_tag="${nvidia_container_runtime_version}${nvidia_container_runtime_tag:+~${nvidia_container_runtime_tag}}"
+
+# Get version for nvidia-docker
+nvidia_docker_version=$(grep -m 1 "^LIB_VERSION := " ${NVIDIA_DOCKER_ROOT}/Makefile | sed -e 's/LIB_VERSION :=[[:space:]]\(.*\)[[:space:]]*/\1/')
+nvidia_docker_tag=$(grep -m 1 "^LIB_TAG .= " ${NVIDIA_DOCKER_ROOT}/Makefile | sed -e 's/LIB_TAG .=[[:space:]]\(.*\)[[:space:]]*/\1/')
+nvidia_docker_version_tag="${nvidia_docker_version}${nvidia_docker_tag:+~${nvidia_docker_tag}}"
+
+
+echo "LIBNVIDIA_CONTAINER_VERSION=${libnvidia_container_version_tag}"
+echo "NVIDIA_CONTAINER_TOOLKIT_VERSION=${nvidia_container_toolkit_version}"
+echo "NVIDIA_CONTAINER_TOOLKIT_TAG=${nvidia_container_toolkit_tag}"
+if [[ "${libnvidia_container_version_tag}" != "${nvidia_container_toolkit_version_tag}" ]]; then
+    >&2 echo "WARNING: The libnvidia-container and nvidia-container-toolkit versions do not match"
+fi
+echo "NVIDIA_CONTAINER_RUNTIME_VERSION=${nvidia_container_runtime_version}"
+echo "NVIDIA_CONTAINER_RUNTIME_TAG=${nvidia_container_runtime_tag}"
+echo "NVIDIA_DOCKER_VERSION=${nvidia_docker_version}"
+echo "NVIDIA_DOCKER_TAG=${nvidia_docker_tag}"

scripts/release.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to build the packages for the components of the NVIDIA
+# Container Stack. These include the nvidia-container-toolkit in this repository
+# as well as the components included in the third_party folder.
+# All required packages are generated in the specified dist folder.
+
+set -e -x
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
+
+# This list represents the distribution-architecture pairs that are actually published
+# to the relevant repositories. This targets forwarded to the build-all-components script
+# can be overridden by specifying command line arguments.
+all=(
+    amazonlinux1-x86_64
+    amazonlinux2-aarch64
+    amazonlinux2-x86_64
+    centos7-ppc64le
+    centos7-x86_64
+    centos8-aarch64
+    centos8-ppc64le
+    centos8-x86_64
+    debian10-amd64
+    debian9-amd64
+    opensuse-leap15.1-x86_64
+    ubuntu16.04-amd64
+    ubuntu16.04-ppc64le
+    ubuntu18.04-amd64
+    ubuntu18.04-arm64
+    ubuntu18.04-ppc64le
+)
+
+if [[ $# -gt 0 ]]; then
+    targets=($*)
+else
+    targets=${all[@]}
+fi
+
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+export NVIDIA_CONTAINER_TOOLKIT_VERSION
+export NVIDIA_CONTAINER_TOOLKIT_TAG
+
+for target in ${targets[@]}; do
+    ${SCRIPTS_DIR}/build-all-components.sh ${target}
+done

test/bin/nvidia-container-runtime-hook

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo mock hook

test/bin/runc

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo mock runc

test/container/common.sh

@@ -0,0 +1,117 @@
+#! /bin/bash
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readonly CRIO_HOOKS_DIR="/usr/share/containers/oci/hooks.d"
+readonly CRIO_HOOK_FILENAME="oci-nvidia-hook.json"
+
+# shellcheck disable=SC2015
+[ -t 2 ] && readonly LOG_TTY=1 || readonly LOG_NO_TTY=1
+
+if [ "${LOG_TTY-0}" -eq 1 ] && [ "$(tput colors)" -ge 15 ]; then
+	readonly FMT_BOLD=$(tput bold)
+	readonly FMT_RED=$(tput setaf 1)
+	readonly FMT_YELLOW=$(tput setaf 3)
+	readonly FMT_BLUE=$(tput setaf 12)
+	readonly FMT_CLEAR=$(tput sgr0)
+fi
+
+log() {
+	local -r level="$1"; shift
+	local -r message="$*"
+
+	local fmt_on="${FMT_CLEAR-}"
+	local -r fmt_off="${FMT_CLEAR-}"
+
+	case "${level}" in
+		INFO)  fmt_on="${FMT_BLUE-}" ;;
+		WARN)  fmt_on="${FMT_YELLOW-}" ;;
+		ERROR) fmt_on="${FMT_RED-}" ;;
+	esac
+	printf "%s[%s]%s %b\n" "${fmt_on}" "${level}" "${fmt_off}" "${message}" >&2
+}
+
+with_retry() {
+	local max_attempts="$1"
+	local delay="$2"
+	local count=0
+	local rc
+	shift 2
+
+	while true; do
+		set +e
+		"$@"; rc="$?"
+		set -e
+
+		count="$((count+1))"
+
+		if [[ "${rc}" -eq 0 ]]; then
+			return 0
+		fi
+
+		if [[ "${max_attempts}" -le 0 ]] || [[ "${count}" -lt "${max_attempts}" ]]; then
+			sleep "${delay}"
+		else
+			break
+		fi
+	done
+
+	return 1
+}
+
+testing::setup() {
+	cp -Rp ${basedir}/shared ${shared_dir}
+	mkdir -p "${shared_dir}/etc/containerd"
+	mkdir -p "${shared_dir}/etc/docker"
+	mkdir -p "${shared_dir}/run/docker/containerd"
+	mkdir -p "${shared_dir}/run/nvidia"
+	mkdir -p "${shared_dir}/usr/local/nvidia"
+	mkdir -p "${shared_dir}${CRIO_HOOKS_DIR}"
+}
+
+testing::cleanup() {
+	if [[ "${CLEANUP}" == "false" ]]; then
+		echo "Skipping cleanup: CLEANUP=${CLEANUP}"
+		return 0
+	fi
+	if [[ -e "${shared_dir}" ]]; then
+		docker run --rm \
+			-v "${shared_dir}:/work" \
+			alpine sh -c 'rm -rf /work/*'
+		rmdir "${shared_dir}"
+	fi
+
+	if [[ "${test_cases:-""}" == "" ]]; then
+		echo "No test cases defined. Skipping test case cleanup"
+		return 0
+	fi
+
+	for tc in ${test_cases}; do
+		testing::${tc}::cleanup
+	done
+}
+
+testing::docker_run::toolkit::shell() {
+	docker run --rm --privileged \
+		--entrypoint sh \
+		-v "${shared_dir}/etc/containerd:/etc/containerd" \
+		-v "${shared_dir}/etc/docker:/etc/docker" \
+		-v "${shared_dir}/run/docker/containerd:/run/docker/containerd" \
+		-v "${shared_dir}/run/nvidia:/run/nvidia" \
+		-v "${shared_dir}/usr/local/nvidia:/usr/local/nvidia" \
+		-v "${shared_dir}${CRIO_HOOKS_DIR}:${CRIO_HOOKS_DIR}" \
+		"${toolkit_container_image}" "-c" "$*"
+}
+
+

test/container/containerd_test.sh

@@ -0,0 +1,147 @@
+#! /bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readonly containerd_dind_ctr="container-config-containerd-dind-ctr-name"
+readonly containerd_test_ctr="container-config-containerd-test-ctr-name"
+readonly containerd_dind_socket="/run/nvidia/docker.sock"
+readonly containerd_dind_containerd_dir="/run/docker/containerd"
+
+testing::containerd::dind::setup() {
+	# Docker creates /etc/docker when starting
+	# by default there isn't any config in this directory (even after the daemon starts)
+	docker run -d --rm --privileged \
+		-v "${shared_dir}/etc/docker:/etc/docker" \
+		-v "${shared_dir}/run/nvidia:/run/nvidia" \
+		-v "${shared_dir}/usr/local/nvidia:/usr/local/nvidia" \
+		-v "${shared_dir}/run/docker/containerd:/run/docker/containerd" \
+		--name "${containerd_dind_ctr}" \
+		docker:stable-dind -H unix://${containerd_dind_socket}
+}
+
+testing::containerd::dind::exec() {
+	docker exec "${containerd_dind_ctr}" sh -c "$*"
+}
+
+testing::containerd::toolkit::run() {
+	local version=${1}
+
+	# We run ctr image list to ensure that containerd has successfully started in the docker-in-docker container
+	with_retry 5 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image list -q"
+
+	# Ensure that we can run some non GPU containers from within dind
+	with_retry 3 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image pull nvcr.io/nvidia/cuda:11.1-base; \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock run --rm --runtime=io.containerd.runtime.v1.linux nvcr.io/nvidia/cuda:11.1-base cuda echo foo"
+
+	# Share the volumes so that we can edit the config file and point to the new runtime
+	# Share the pid so that we can ask docker to reload its config
+	docker run --rm --privileged \
+		--volumes-from "${containerd_dind_ctr}" \
+		-v "${shared_dir}/etc/containerd/config_${version}.toml:${containerd_dind_containerd_dir}/containerd.toml" \
+		--pid "container:${containerd_dind_ctr}" \
+		-e "RUNTIME=containerd" \
+		-e "RUNTIME_ARGS=--config=${containerd_dind_containerd_dir}/containerd.toml --socket=${containerd_dind_containerd_dir}/containerd.sock" \
+		--name "${containerd_test_ctr}" \
+		"${toolkit_container_image}" "/usr/local/nvidia" "--no-daemon"
+
+	# We run ctr image list to ensure that containerd has successfully started in the docker-in-docker container
+	with_retry 5 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image list -q"
+
+	# Ensure that we haven't broken non GPU containers
+	with_retry 3 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image pull nvcr.io/nvidia/cuda:11.1-base; \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock run --rm --runtime=io.containerd.runtime.v1.linux nvcr.io/nvidia/cuda:11.1-base cuda echo foo"
+}
+
+# This test runs containerd setup and containerd cleanup in succession to ensure that the
+# config is restored correctly.
+testing::containerd::toolkit::test_config() {
+	local version=${1}
+
+	# We run ctr image list to ensure that containerd has successfully started in the docker-in-docker container
+	with_retry 5 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image list -q"
+
+	local input_config="${shared_dir}/etc/containerd/config_${version}.toml"
+	local output_config="${shared_dir}/output/config_${version}.toml"
+	local output_dir=$(dirname ${output_config})
+
+	mkdir -p ${output_dir}
+	cp -p "${input_config}" "${output_config}"
+
+	docker run --rm --privileged \
+		--volumes-from "${containerd_dind_ctr}" \
+		-v "${output_dir}:${output_dir}" \
+		--name "${containerd_test_ctr}" \
+		--entrypoint sh \
+		"${toolkit_container_image}" -c "containerd setup \
+			--config=${output_config} \
+			--socket=${containerd_dind_containerd_dir}/containerd.sock \
+			--restart-mode=NONE \
+				/usr/local/nvidia/toolkit"
+
+	# As a basic test we check that the config has changed
+	diff "${input_config}" "${output_config}" || test ${?} -ne 0
+	grep -q -E "^version = \d" "${output_config}"
+	grep -q -E "default_runtime_name = \"nvidia\"" "${output_config}"
+
+	docker run --rm --privileged \
+		--volumes-from "${containerd_dind_ctr}" \
+		-v "${output_dir}:${output_dir}" \
+		--name "${containerd_test_ctr}" \
+		--entrypoint sh \
+		"${toolkit_container_image}" -c "containerd cleanup \
+					--config=${output_config} \
+			--socket=${containerd_dind_containerd_dir}/containerd.sock \
+			--restart-mode=NONE \
+				/usr/local/nvidia/toolkit"
+
+	if [[ -s "${input_config}" ]]; then
+		# Compare the input and output config. These should be the same.
+		diff "${input_config}" "${output_config}" || true
+	else
+		# If the input config is empty, the output should not exist.
+		test ! -e "${output_config}"
+	fi
+}
+
+testing::containerd::main() {
+	testing::containerd::dind::setup
+
+	testing::containerd::toolkit::test_config empty
+	testing::containerd::toolkit::test_config v1
+	testing::containerd::toolkit::test_config v2
+
+	testing::containerd::cleanup
+
+	testing::containerd::dind::setup
+	testing::containerd::toolkit::run empty
+	testing::containerd::cleanup
+
+	testing::containerd::dind::setup
+	testing::containerd::toolkit::run v1
+	testing::containerd::cleanup
+
+	testing::containerd::dind::setup
+	testing::containerd::toolkit::run v2
+	testing::containerd::cleanup
+}
+
+testing::containerd::cleanup() {
+	docker kill "${containerd_dind_ctr}" &> /dev/null || true
+	docker kill "${containerd_test_ctr}" &> /dev/null || true
+}

test/container/crio_test.sh

@@ -0,0 +1,42 @@
+#! /bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+testing::crio::hook_created() {
+	testing::docker_run::toolkit::shell 'crio setup /run/nvidia/toolkit'
+
+	test ! -z "$(ls -A "${shared_dir}${CRIO_HOOKS_DIR}")"
+
+	cat "${shared_dir}${CRIO_HOOKS_DIR}/${CRIO_HOOK_FILENAME}" | \
+		jq -r '.hook.path' | grep -q "/run/nvidia/toolkit/"
+	test $? -eq 0
+	cat "${shared_dir}${CRIO_HOOKS_DIR}/${CRIO_HOOK_FILENAME}" | \
+		jq -r '.hook.env[0]' | grep -q ":/run/nvidia/toolkit"
+	test $? -eq 0
+}
+
+testing::crio::hook_cleanup() {
+	testing::docker_run::toolkit::shell 'crio cleanup'
+
+	test -z "$(ls -A "${shared_dir}${CRIO_HOOKS_DIR}")"
+}
+
+testing::crio::main() {
+	testing::crio::hook_created
+	testing::crio::hook_cleanup
+}
+
+testing::crio::cleanup() {
+	:
+}

test/container/docker_test.sh

@@ -0,0 +1,57 @@
+#! /bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readonly docker_dind_ctr="container-config-docker-dind-ctr-name"
+readonly docker_test_ctr="container-config-docker-test-ctr-name"
+readonly docker_dind_socket="/run/nvidia/docker.sock"
+
+testing::docker::dind::setup() {
+	# Docker creates /etc/docker when starting
+	# by default there isn't any config in this directory (even after the daemon starts)
+	docker run -d --rm --privileged \
+		-v "${shared_dir}/etc/docker:/etc/docker" \
+		-v "${shared_dir}/run/nvidia:/run/nvidia" \
+		-v "${shared_dir}/usr/local/nvidia:/usr/local/nvidia" \
+		--name "${docker_dind_ctr}" \
+		docker:stable-dind -H unix://${docker_dind_socket}
+}
+
+testing::docker::dind::exec() {
+	docker exec "${docker_dind_ctr}" sh -c "$*"
+}
+
+testing::docker::toolkit::run() {
+	# Share the volumes so that we can edit the config file and point to the new runtime
+	# Share the pid so that we can ask docker to reload its config
+	docker run -d --rm --privileged \
+		--volumes-from "${docker_dind_ctr}" \
+		--pid "container:${docker_dind_ctr}" \
+		-e "RUNTIME_ARGS=--socket ${docker_dind_socket}" \
+		--name "${docker_test_ctr}" \
+		"${toolkit_container_image}" "/usr/local/nvidia" "--no-daemon"
+
+	# Ensure that we haven't broken non GPU containers
+	with_retry 3 5s testing::docker::dind::exec docker run -t alpine echo foo
+}
+
+testing::docker::main() {
+	testing::docker::dind::setup
+	testing::docker::toolkit::run
+}
+
+testing::docker::cleanup() {
+	docker kill "${docker_dind_ctr}" &> /dev/null || true
+	docker kill "${docker_test_ctr}" &> /dev/null || true
+}

test/container/main.sh

@@ -0,0 +1,77 @@
+#! /bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eEuo pipefail
+shopt -s lastpipe
+
+readonly basedir="$(dirname "$(realpath "$0")")"
+source "${basedir}/common.sh"
+
+source "${basedir}/toolkit_test.sh"
+source "${basedir}/docker_test.sh"
+source "${basedir}/crio_test.sh"
+source "${basedir}/containerd_test.sh"
+
+: ${CLEANUP:=true}
+
+usage() {
+	cat >&2 <<EOF
+Usage: $0 COMMAND [ARG...]
+
+Commands:
+  run SHARED_DIR TOOLKIT_CONTAINER_IMAGE [-c | --no-cleanup-on-error ]
+  clean SHARED_DIR
+EOF
+}
+
+if [ $# -lt 2 ]; then usage; exit 1; fi
+
+# We defined shared_dir here so that it can be used in cleanup
+readonly command=${1}; shift
+readonly shared_dir="${1}"; shift;
+
+case "${command}" in
+	clean) testing::cleanup; exit 0;;
+	run) ;;
+	*) usage; exit 0;;
+esac
+
+if [ $# -eq 0 ]; then usage; exit 1; fi
+
+readonly toolkit_container_image="${1}"; shift
+
+options=$(getopt -l no-cleanup-on-error -o c -- "$@")
+if [[ "$?" -ne 0 ]]; then usage; exit 1; fi
+
+# set options to positional parameters
+eval set -- "${options}"
+for opt in ${options}; do
+	case "${opt}" in
+	c | --no-cleanup-on-error) CLEANUP=false; shift;;
+	--) shift; break;;
+	esac
+done
+
+trap '"$CLEANUP" && testing::cleanup' ERR
+
+readonly test_cases="${TEST_CASES:-toolkit docker crio containerd}"
+
+testing::cleanup
+for tc in ${test_cases}; do
+	log INFO "=================Testing ${tc}================="
+	testing::setup
+	testing::${tc}::main "$@"
+	testing::cleanup
+done

test/container/shared/etc/containerd/config_v1.toml

@@ -0,0 +1,92 @@
+oom_score = 0
+root = "/var/lib/containerd"
+state = "/run/containerd"
+
+[cgroup]
+  path = ""
+
+[debug]
+  address = "/var/run/docker/containerd/containerd-debug.sock"
+  gid = 0
+  level = ""
+  uid = 0
+
+[grpc]
+  address = "/var/run/docker/containerd/containerd.sock"
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+  uid = 0
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[plugins]
+
+  [plugins.cgroups]
+    no_prometheus = false
+
+  [plugins.cri]
+    disable_proc_mount = false
+    enable_selinux = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    sandbox_image = "k8s.gcr.io/pause:3.1"
+    stats_collect_period = 10
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    systemd_cgroup = false
+
+    [plugins.cri.cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+
+    [plugins.cri.containerd]
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins.cri.containerd.default_runtime]
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = "io.containerd.runtime.v1.linux"
+
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = ""
+
+    [plugins.cri.registry]
+
+      [plugins.cri.registry.mirrors]
+
+        [plugins.cri.registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+
+    [plugins.cri.x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+
+  [plugins.diff-service]
+    default = ["walking"]
+
+  [plugins.linux]
+    no_shim = false
+    runtime = "runc"
+    runtime_root = "/var/lib/docker/runc"
+    shim = "containerd-shim"
+    shim_debug = false
+
+  [plugins.opt]
+    path = "/opt/containerd"
+
+  [plugins.restart]
+    interval = "10s"
+
+  [plugins.scheduler]
+    deletion_threshold = 0
+    mutation_threshold = 100
+    pause_threshold = 0.02
+    schedule_delay = "0s"
+    startup_delay = "100ms"

test/container/shared/etc/containerd/config_v2.toml

@@ -0,0 +1,139 @@
+disabled_plugins = []
+oom_score = 0
+plugin_dir = ""
+required_plugins = []
+root = "/var/lib/containerd"
+state = "/run/containerd"
+version = 2
+
+[cgroup]
+  path = ""
+
+[debug]
+  address = "/var/run/docker/containerd/containerd-debug.sock"
+  gid = 0
+  level = ""
+  uid = 0
+
+[grpc]
+  address = "/var/run/docker/containerd/containerd.sock"
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+  tcp_address = ""
+  tcp_tls_cert = ""
+  tcp_tls_key = ""
+  uid = 0
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[plugins]
+
+  [plugins."io.containerd.gc.v1.scheduler"]
+    deletion_threshold = 0
+    mutation_threshold = 100
+    pause_threshold = 0.02
+    schedule_delay = "0s"
+    startup_delay = "100ms"
+
+  [plugins."io.containerd.grpc.v1.cri"]
+    disable_apparmor = false
+    disable_cgroup = false
+    disable_proc_mount = false
+    disable_tcp_service = true
+    enable_selinux = false
+    enable_tls_streaming = false
+    max_concurrent_downloads = 3
+    max_container_log_line_size = 16384
+    restrict_oom_score_adj = false
+    sandbox_image = "k8s.gcr.io/pause:3.1"
+    stats_collect_period = 10
+    stream_idle_timeout = "4h0m0s"
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    systemd_cgroup = false
+
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+      max_conf_num = 1
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        privileged_without_host_devices = false
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = ""
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v1"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
+        privileged_without_host_devices = false
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = ""
+
+    [plugins."io.containerd.grpc.v1.cri".registry]
+
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+
+    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+
+  [plugins."io.containerd.internal.v1.opt"]
+    path = "/opt/containerd"
+
+  [plugins."io.containerd.internal.v1.restart"]
+    interval = "10s"
+
+  [plugins."io.containerd.metadata.v1.bolt"]
+    content_sharing_policy = "shared"
+
+  [plugins."io.containerd.monitor.v1.cgroups"]
+    no_prometheus = false
+
+  [plugins."io.containerd.runtime.v1.linux"]
+    no_shim = false
+    runtime = "runc"
+    runtime_root = "/var/lib/docker/runc"
+    shim = "containerd-shim"
+    shim_debug = false
+
+  [plugins."io.containerd.runtime.v2.task"]
+    platforms = ["linux/amd64"]
+
+  [plugins."io.containerd.service.v1.diff-service"]
+    default = ["walking"]
+
+  [plugins."io.containerd.snapshotter.v1.devmapper"]
+    base_image_size = ""
+    pool_name = ""
+    root_path = ""
+
+[timeouts]
+  "io.containerd.timeout.shim.cleanup" = "5s"
+  "io.containerd.timeout.shim.load" = "5s"
+  "io.containerd.timeout.shim.shutdown" = "3s"
+  "io.containerd.timeout.task.state" = "2s"
+
+[ttrpc]
+  address = ""
+  gid = 0
+  uid = 0

test/container/shared/etc/docker/daemon.json

@@ -0,0 +1,3 @@
+{
+  "registry-mirrors": ["https://mirror.gcr.io"]
+}

test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so

@@ -0,0 +1 @@
+# This is a dummy lib file to test nvidia-runtime-experimental

test/container/toolkit_test.sh

@@ -0,0 +1,79 @@
+#! /bin/bash
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+testing::toolkit::install() {
+	local -r uid=$(id -u)
+	local -r gid=$(id -g)
+
+	local READLINK="readlink"
+	local -r platform=$(uname)
+	if [[ "${platform}" == "Darwin" ]]; then
+		READLINK="greadlink"
+	fi
+
+	testing::docker_run::toolkit::shell 'toolkit install /usr/local/nvidia/toolkit'
+	docker run --rm -v "${shared_dir}:/work" alpine sh -c "chown -R ${uid}:${gid} /work/"
+
+	# Ensure toolkit dir is correctly setup
+	test ! -z "$(ls -A "${shared_dir}/usr/local/nvidia/toolkit")"
+
+	test -L "${shared_dir}/usr/local/nvidia/toolkit/libnvidia-container.so.1"
+	test -e "$(${READLINK} -f "${shared_dir}/usr/local/nvidia/toolkit/libnvidia-container.so.1")"
+
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-cli"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-toolkit"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+
+	grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+	grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-cli.real"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-toolkit.real"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.real"
+
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
+
+	grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
+	grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
+	grep -q -E "LD_LIBRARY_PATH=/run/nvidia/driver/usr/lib64:\\\$LD_LIBRARY_PATH " "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
+
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
+
+	# Ensure that the config file has the required contents.
+	# NOTE: This assumes that RUN_DIR is '/run/nvidia'
+	local -r nvidia_run_dir="/run/nvidia"
+	grep -q -E "^\s*ldconfig = \"@${nvidia_run_dir}/driver/sbin/ldconfig(.real)?\"" "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
+	grep -q -E "^\s*root = \"${nvidia_run_dir}/driver\"" "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
+	grep -q -E "^\s*path = \"/usr/local/nvidia/toolkit/nvidia-container-cli\"" "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
+}
+
+testing::toolkit::delete() {
+	testing::docker_run::toolkit::shell 'mkdir -p /usr/local/nvidia/delete-toolkit'
+	testing::docker_run::toolkit::shell 'touch /usr/local/nvidia/delete-toolkit/test.file'
+	testing::docker_run::toolkit::shell 'toolkit delete /usr/local/nvidia/delete-toolkit'
+
+	test ! -z "$(ls -A "${shared_dir}/usr/local/nvidia")"
+	test ! -e "${shared_dir}/usr/local/nvidia/delete-toolkit"
+}
+
+testing::toolkit::main() {
+	testing::toolkit::install
+	testing::toolkit::delete
+}
+
+testing::toolkit::cleanup() {
+	:
+}

test/input/config.clone3.json

@@ -0,0 +1,784 @@
+{
+  "ociVersion": "1.0.2-dev",
+  "process": {
+    "terminal": true,
+    "user": {
+      "uid": 0,
+      "gid": 0
+    },
+    "args": [
+      "sleep",
+      "60"
+    ],
+    "env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+      "HOSTNAME=8de5efc6a95c",
+      "TERM=xterm"
+    ],
+    "cwd": "/",
+    "capabilities": {
+      "bounding": [
+        "CAP_CHOWN",
+        "CAP_DAC_OVERRIDE",
+        "CAP_FSETID",
+        "CAP_FOWNER",
+        "CAP_MKNOD",
+        "CAP_NET_RAW",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETFCAP",
+        "CAP_SETPCAP",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_SYS_CHROOT",
+        "CAP_KILL",
+        "CAP_AUDIT_WRITE"
+      ],
+      "effective": [
+        "CAP_CHOWN",
+        "CAP_DAC_OVERRIDE",
+        "CAP_FSETID",
+        "CAP_FOWNER",
+        "CAP_MKNOD",
+        "CAP_NET_RAW",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETFCAP",
+        "CAP_SETPCAP",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_SYS_CHROOT",
+        "CAP_KILL",
+        "CAP_AUDIT_WRITE"
+      ],
+      "inheritable": [
+        "CAP_CHOWN",
+        "CAP_DAC_OVERRIDE",
+        "CAP_FSETID",
+        "CAP_FOWNER",
+        "CAP_MKNOD",
+        "CAP_NET_RAW",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETFCAP",
+        "CAP_SETPCAP",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_SYS_CHROOT",
+        "CAP_KILL",
+        "CAP_AUDIT_WRITE"
+      ],
+      "permitted": [
+        "CAP_CHOWN",
+        "CAP_DAC_OVERRIDE",
+        "CAP_FSETID",
+        "CAP_FOWNER",
+        "CAP_MKNOD",
+        "CAP_NET_RAW",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETFCAP",
+        "CAP_SETPCAP",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_SYS_CHROOT",
+        "CAP_KILL",
+        "CAP_AUDIT_WRITE"
+      ]
+    },
+    "apparmorProfile": "docker-default",
+    "oomScoreAdj": 0
+  },
+  "root": {
+    "path": "/var/lib/docker/overlay2/fbf92f54592ddb439159bc7eb25c865b9347a2d71d63b41b7b4e4a471847c84f/merged"
+  },
+  "hostname": "8de5efc6a95c",
+  "mounts": [
+    {
+      "destination": "/proc",
+      "type": "proc",
+      "source": "proc",
+      "options": [
+        "nosuid",
+        "noexec",
+        "nodev"
+      ]
+    },
+    {
+      "destination": "/dev",
+      "type": "tmpfs",
+      "source": "tmpfs",
+      "options": [
+        "nosuid",
+        "strictatime",
+        "mode=755",
+        "size=65536k"
+      ]
+    },
+    {
+      "destination": "/dev/pts",
+      "type": "devpts",
+      "source": "devpts",
+      "options": [
+        "nosuid",
+        "noexec",
+        "newinstance",
+        "ptmxmode=0666",
+        "mode=0620",
+        "gid=5"
+      ]
+    },
+    {
+      "destination": "/sys",
+      "type": "sysfs",
+      "source": "sysfs",
+      "options": [
+        "nosuid",
+        "noexec",
+        "nodev",
+        "ro"
+      ]
+    },
+    {
+      "destination": "/sys/fs/cgroup",
+      "type": "cgroup",
+      "source": "cgroup",
+      "options": [
+        "ro",
+        "nosuid",
+        "noexec",
+        "nodev"
+      ]
+    },
+    {
+      "destination": "/dev/mqueue",
+      "type": "mqueue",
+      "source": "mqueue",
+      "options": [
+        "nosuid",
+        "noexec",
+        "nodev"
+      ]
+    },
+    {
+      "destination": "/dev/shm",
+      "type": "tmpfs",
+      "source": "shm",
+      "options": [
+        "nosuid",
+        "noexec",
+        "nodev",
+        "mode=1777",
+        "size=67108864"
+      ]
+    },
+    {
+      "destination": "/etc/resolv.conf",
+      "type": "bind",
+      "source": "/var/lib/docker/containers/8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6/resolv.conf",
+      "options": [
+        "rbind",
+        "rprivate"
+      ]
+    },
+    {
+      "destination": "/etc/hostname",
+      "type": "bind",
+      "source": "/var/lib/docker/containers/8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6/hostname",
+      "options": [
+        "rbind",
+        "rprivate"
+      ]
+    },
+    {
+      "destination": "/etc/hosts",
+      "type": "bind",
+      "source": "/var/lib/docker/containers/8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6/hosts",
+      "options": [
+        "rbind",
+        "rprivate"
+      ]
+    }
+  ],
+  "hooks": {
+    "prestart": [
+      {
+        "path": "/proc/593/exe",
+        "args": [
+          "libnetwork-setkey",
+          "-exec-root=/var/run/docker",
+          "8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6",
+          "9967b9f7c4d4"
+        ]
+      }
+    ]
+  },
+  "linux": {
+    "sysctl": {
+      "net.ipv4.ip_unprivileged_port_start": "0"
+    },
+    "resources": {
+      "devices": [
+        {
+          "allow": false,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 1,
+          "minor": 5,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 1,
+          "minor": 3,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 1,
+          "minor": 9,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 1,
+          "minor": 8,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 5,
+          "minor": 0,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 5,
+          "minor": 1,
+          "access": "rwm"
+        },
+        {
+          "allow": false,
+          "type": "c",
+          "major": 10,
+          "minor": 229,
+          "access": "rwm"
+        }
+      ],
+      "memory": {
+        "disableOOMKiller": false
+      },
+      "cpu": {
+        "shares": 0
+      },
+      "blockIO": {
+        "weight": 0
+      }
+    },
+    "cgroupsPath": "/docker/8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6",
+    "namespaces": [
+      {
+        "type": "mount"
+      },
+      {
+        "type": "network"
+      },
+      {
+        "type": "uts"
+      },
+      {
+        "type": "pid"
+      },
+      {
+        "type": "ipc"
+      }
+    ],
+    "seccomp": {
+      "defaultAction": "SCMP_ACT_ERRNO",
+      "architectures": [
+        "SCMP_ARCH_X86_64",
+        "SCMP_ARCH_X86",
+        "SCMP_ARCH_X32"
+      ],
+      "syscalls": [
+        {
+          "names": [
+            "accept",
+            "accept4",
+            "access",
+            "adjtimex",
+            "alarm",
+            "bind",
+            "brk",
+            "capget",
+            "capset",
+            "chdir",
+            "chmod",
+            "chown",
+            "chown32",
+            "clock_adjtime",
+            "clock_adjtime64",
+            "clock_getres",
+            "clock_getres_time64",
+            "clock_gettime",
+            "clock_gettime64",
+            "clock_nanosleep",
+            "clock_nanosleep_time64",
+            "close",
+            "close_range",
+            "connect",
+            "copy_file_range",
+            "creat",
+            "dup",
+            "dup2",
+            "dup3",
+            "epoll_create",
+            "epoll_create1",
+            "epoll_ctl",
+            "epoll_ctl_old",
+            "epoll_pwait",
+            "epoll_pwait2",
+            "epoll_wait",
+            "epoll_wait_old",
+            "eventfd",
+            "eventfd2",
+            "execve",
+            "execveat",
+            "exit",
+            "exit_group",
+            "faccessat",
+            "faccessat2",
+            "fadvise64",
+            "fadvise64_64",
+            "fallocate",
+            "fanotify_mark",
+            "fchdir",
+            "fchmod",
+            "fchmodat",
+            "fchown",
+            "fchown32",
+            "fchownat",
+            "fcntl",
+            "fcntl64",
+            "fdatasync",
+            "fgetxattr",
+            "flistxattr",
+            "flock",
+            "fork",
+            "fremovexattr",
+            "fsetxattr",
+            "fstat",
+            "fstat64",
+            "fstatat64",
+            "fstatfs",
+            "fstatfs64",
+            "fsync",
+            "ftruncate",
+            "ftruncate64",
+            "futex",
+            "futex_time64",
+            "futimesat",
+            "getcpu",
+            "getcwd",
+            "getdents",
+            "getdents64",
+            "getegid",
+            "getegid32",
+            "geteuid",
+            "geteuid32",
+            "getgid",
+            "getgid32",
+            "getgroups",
+            "getgroups32",
+            "getitimer",
+            "getpeername",
+            "getpgid",
+            "getpgrp",
+            "getpid",
+            "getppid",
+            "getpriority",
+            "getrandom",
+            "getresgid",
+            "getresgid32",
+            "getresuid",
+            "getresuid32",
+            "getrlimit",
+            "get_robust_list",
+            "getrusage",
+            "getsid",
+            "getsockname",
+            "getsockopt",
+            "get_thread_area",
+            "gettid",
+            "gettimeofday",
+            "getuid",
+            "getuid32",
+            "getxattr",
+            "inotify_add_watch",
+            "inotify_init",
+            "inotify_init1",
+            "inotify_rm_watch",
+            "io_cancel",
+            "ioctl",
+            "io_destroy",
+            "io_getevents",
+            "io_pgetevents",
+            "io_pgetevents_time64",
+            "ioprio_get",
+            "ioprio_set",
+            "io_setup",
+            "io_submit",
+            "io_uring_enter",
+            "io_uring_register",
+            "io_uring_setup",
+            "ipc",
+            "kill",
+            "lchown",
+            "lchown32",
+            "lgetxattr",
+            "link",
+            "linkat",
+            "listen",
+            "listxattr",
+            "llistxattr",
+            "_llseek",
+            "lremovexattr",
+            "lseek",
+            "lsetxattr",
+            "lstat",
+            "lstat64",
+            "madvise",
+            "membarrier",
+            "memfd_create",
+            "mincore",
+            "mkdir",
+            "mkdirat",
+            "mknod",
+            "mknodat",
+            "mlock",
+            "mlock2",
+            "mlockall",
+            "mmap",
+            "mmap2",
+            "mprotect",
+            "mq_getsetattr",
+            "mq_notify",
+            "mq_open",
+            "mq_timedreceive",
+            "mq_timedreceive_time64",
+            "mq_timedsend",
+            "mq_timedsend_time64",
+            "mq_unlink",
+            "mremap",
+            "msgctl",
+            "msgget",
+            "msgrcv",
+            "msgsnd",
+            "msync",
+            "munlock",
+            "munlockall",
+            "munmap",
+            "nanosleep",
+            "newfstatat",
+            "_newselect",
+            "open",
+            "openat",
+            "openat2",
+            "pause",
+            "pidfd_open",
+            "pidfd_send_signal",
+            "pipe",
+            "pipe2",
+            "poll",
+            "ppoll",
+            "ppoll_time64",
+            "prctl",
+            "pread64",
+            "preadv",
+            "preadv2",
+            "prlimit64",
+            "pselect6",
+            "pselect6_time64",
+            "pwrite64",
+            "pwritev",
+            "pwritev2",
+            "read",
+            "readahead",
+            "readlink",
+            "readlinkat",
+            "readv",
+            "recv",
+            "recvfrom",
+            "recvmmsg",
+            "recvmmsg_time64",
+            "recvmsg",
+            "remap_file_pages",
+            "removexattr",
+            "rename",
+            "renameat",
+            "renameat2",
+            "restart_syscall",
+            "rmdir",
+            "rseq",
+            "rt_sigaction",
+            "rt_sigpending",
+            "rt_sigprocmask",
+            "rt_sigqueueinfo",
+            "rt_sigreturn",
+            "rt_sigsuspend",
+            "rt_sigtimedwait",
+            "rt_sigtimedwait_time64",
+            "rt_tgsigqueueinfo",
+            "sched_getaffinity",
+            "sched_getattr",
+            "sched_getparam",
+            "sched_get_priority_max",
+            "sched_get_priority_min",
+            "sched_getscheduler",
+            "sched_rr_get_interval",
+            "sched_rr_get_interval_time64",
+            "sched_setaffinity",
+            "sched_setattr",
+            "sched_setparam",
+            "sched_setscheduler",
+            "sched_yield",
+            "seccomp",
+            "select",
+            "semctl",
+            "semget",
+            "semop",
+            "semtimedop",
+            "semtimedop_time64",
+            "send",
+            "sendfile",
+            "sendfile64",
+            "sendmmsg",
+            "sendmsg",
+            "sendto",
+            "setfsgid",
+            "setfsgid32",
+            "setfsuid",
+            "setfsuid32",
+            "setgid",
+            "setgid32",
+            "setgroups",
+            "setgroups32",
+            "setitimer",
+            "setpgid",
+            "setpriority",
+            "setregid",
+            "setregid32",
+            "setresgid",
+            "setresgid32",
+            "setresuid",
+            "setresuid32",
+            "setreuid",
+            "setreuid32",
+            "setrlimit",
+            "set_robust_list",
+            "setsid",
+            "setsockopt",
+            "set_thread_area",
+            "set_tid_address",
+            "setuid",
+            "setuid32",
+            "setxattr",
+            "shmat",
+            "shmctl",
+            "shmdt",
+            "shmget",
+            "shutdown",
+            "sigaltstack",
+            "signalfd",
+            "signalfd4",
+            "sigprocmask",
+            "sigreturn",
+            "socket",
+            "socketcall",
+            "socketpair",
+            "splice",
+            "stat",
+            "stat64",
+            "statfs",
+            "statfs64",
+            "statx",
+            "symlink",
+            "symlinkat",
+            "sync",
+            "sync_file_range",
+            "syncfs",
+            "sysinfo",
+            "tee",
+            "tgkill",
+            "time",
+            "timer_create",
+            "timer_delete",
+            "timer_getoverrun",
+            "timer_gettime",
+            "timer_gettime64",
+            "timer_settime",
+            "timer_settime64",
+            "timerfd_create",
+            "timerfd_gettime",
+            "timerfd_gettime64",
+            "timerfd_settime",
+            "timerfd_settime64",
+            "times",
+            "tkill",
+            "truncate",
+            "truncate64",
+            "ugetrlimit",
+            "umask",
+            "uname",
+            "unlink",
+            "unlinkat",
+            "utime",
+            "utimensat",
+            "utimensat_time64",
+            "utimes",
+            "vfork",
+            "vmsplice",
+            "wait4",
+            "waitid",
+            "waitpid",
+            "write",
+            "writev"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        },
+        {
+          "names": [
+            "ptrace"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 0,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 8,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 131072,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 131080,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 4294967295,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "arch_prctl"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        },
+        {
+          "names": [
+            "modify_ldt"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        },
+        {
+          "names": [
+            "clone"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 2114060288,
+              "op": "SCMP_CMP_MASKED_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "clone3"
+          ],
+          "action": "SCMP_ACT_ERRNO",
+          "errnoRet": 38
+        },
+        {
+          "names": [
+            "chroot"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        }
+      ]
+    },
+    "maskedPaths": [
+      "/proc/asound",
+      "/proc/acpi",
+      "/proc/kcore",
+      "/proc/keys",
+      "/proc/latency_stats",
+      "/proc/timer_list",
+      "/proc/timer_stats",
+      "/proc/sched_debug",
+      "/proc/scsi",
+      "/sys/firmware"
+    ],
+    "readonlyPaths": [
+      "/proc/bus",
+      "/proc/fs",
+      "/proc/irq",
+      "/proc/sys",
+      "/proc/sysrq-trigger"
+    ]
+  }
+}

test/input/test_spec.json

@@ -0,0 +1,178 @@
+{
+	"ociVersion": "1.0.1-dev",
+	"process": {
+		"terminal": true,
+		"user": {
+			"uid": 0,
+			"gid": 0
+		},
+		"args": [
+			"sh"
+		],
+		"env": [
+			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+			"TERM=xterm"
+		],
+		"cwd": "/",
+		"capabilities": {
+			"bounding": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"effective": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"inheritable": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"permitted": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			],
+			"ambient": [
+				"CAP_AUDIT_WRITE",
+				"CAP_KILL",
+				"CAP_NET_BIND_SERVICE"
+			]
+		},
+		"rlimits": [
+			{
+				"type": "RLIMIT_NOFILE",
+				"hard": 1024,
+				"soft": 1024
+			}
+		],
+		"noNewPrivileges": true
+	},
+	"root": {
+		"path": "rootfs",
+		"readonly": true
+	},
+	"hostname": "runc",
+	"mounts": [
+		{
+			"destination": "/proc",
+			"type": "proc",
+			"source": "proc"
+		},
+		{
+			"destination": "/dev",
+			"type": "tmpfs",
+			"source": "tmpfs",
+			"options": [
+				"nosuid",
+				"strictatime",
+				"mode=755",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/pts",
+			"type": "devpts",
+			"source": "devpts",
+			"options": [
+				"nosuid",
+				"noexec",
+				"newinstance",
+				"ptmxmode=0666",
+				"mode=0620",
+				"gid=5"
+			]
+		},
+		{
+			"destination": "/dev/shm",
+			"type": "tmpfs",
+			"source": "shm",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"mode=1777",
+				"size=65536k"
+			]
+		},
+		{
+			"destination": "/dev/mqueue",
+			"type": "mqueue",
+			"source": "mqueue",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev"
+			]
+		},
+		{
+			"destination": "/sys",
+			"type": "sysfs",
+			"source": "sysfs",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"ro"
+			]
+		},
+		{
+			"destination": "/sys/fs/cgroup",
+			"type": "cgroup",
+			"source": "cgroup",
+			"options": [
+				"nosuid",
+				"noexec",
+				"nodev",
+				"relatime",
+				"ro"
+			]
+		}
+	],
+	"linux": {
+		"resources": {
+			"devices": [
+				{
+					"allow": false,
+					"access": "rwm"
+				}
+			]
+		},
+		"namespaces": [
+			{
+				"type": "pid"
+			},
+			{
+				"type": "network"
+			},
+			{
+				"type": "ipc"
+			},
+			{
+				"type": "uts"
+			},
+			{
+				"type": "mount"
+			}
+		],
+		"maskedPaths": [
+			"/proc/kcore",
+			"/proc/latency_stats",
+			"/proc/timer_list",
+			"/proc/timer_stats",
+			"/proc/sched_debug",
+			"/sys/firmware",
+			"/proc/scsi"
+		],
+		"readonlyPaths": [
+			"/proc/asound",
+			"/proc/bus",
+			"/proc/fs",
+			"/proc/irq",
+			"/proc/sys",
+			"/proc/sysrq-trigger"
+		]
+	}
+}

test/release/Makefile

@@ -0,0 +1,59 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+WORKFLOW ?= nvidia-docker
+TEST_REPO ?= elezar.github.io
+
+DISTRIBUTIONS := ubuntu18.04 centos8
+
+IMAGE_TARGETS := $(patsubst %,image-%, $(DISTRIBUTIONS))
+RUN_TARGETS := $(patsubst %,run-%, $(DISTRIBUTIONS))
+RELEASE_TARGETS := $(patsubst %,release-%, $(DISTRIBUTIONS))
+LOCAL_TARGETS := $(patsubst %,local-%, $(DISTRIBUTIONS))
+
+.PHONY: $(IMAGE_TARGETS)
+
+image-%: DOCKERFILE = docker/$(*)/Dockerfile
+
+images: $(IMAGE_TARGETS)
+$(IMAGE_TARGETS): image-%:
+	docker build \
+    --build-arg WORKFLOW="$(WORKFLOW)" \
+    --build-arg TEST_REPO="$(TEST_REPO)" \
+    -t nvidia-container-toolkit-repo-test:$(*) \
+    -f $(DOCKERFILE) \
+    $(shell dirname $(DOCKERFILE))
+
+
+%-ubuntu18.04: ARCH = amd64
+%-centos8: ARCH = x86_64
+
+RELEASE_TEST_DIR := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
+PROJECT_ROOT := $(RELEASE_TEST_DIR)/../..
+
+LOCAL_PACKAGE_ROOT := $(PROJECT_ROOT)/dist
+
+local-%: DIST = $(*)
+local-%: LOCAL_REPO_ARGS = -v $(LOCAL_PACKAGE_ROOT)/$(DIST)/$(ARCH):/local-repository
+$(LOCAL_TARGETS): local-%: release-% run-% | release-%
+
+run-%: DIST = $(*)
+$(RUN_TARGETS): run-%:
+	docker run --rm -ti  \
+    $(LOCAL_REPO_ARGS) \
+    nvidia-container-toolkit-repo-test:$(*)
+
+# Ensure that the local package root exists
+$(RELEASE_TARGETS): release-%: $(LOCAL_PACKAGE_ROOT)/$(*)/$(ARCH)
+	$(PROJECT_ROOT)/scripts/release.sh $(*)-$(ARCH)

test/release/README.md

@@ -0,0 +1,107 @@
+# Testing Packaging Workflows
+
+## Building the Docker images:
+
+```bash
+make images
+```
+
+This assumes that the `nvidia-docker` workflow is being tested and that the test packages have been published to the `elezar.github.io` GitHub pages page. These can be overridden
+using make variables.
+
+Valid values for `workflow` are:
+* `nvidia-docker`
+* `nvidia-container-runtime`
+
+This follows the instructions for setting up the [`nvidia-docker` repostitory](https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html#setting-up-nvidia-container-toolkit).
+
+
+## Testing local package changes
+
+Running:
+```bash
+make local-ubuntu18.04
+```
+will build the `ubuntu18.04-amd64` packages for release and launch a docker container with these packages added to a local APT repository.
+
+The various `apt` workflows can then be tested as if the packages were released to the `libnvidia-container` experimental repository.
+
+The `local-centos8` make target is available for testing the `centos8-x86_64` workflows as representative of `yum`-based installation workflows.
+
+
+### Example
+
+In the `centos8`-based container above we see the following `nvidia-docker2` packages available with the `local-repository` disabled:
+
+```bash
+$ yum list --showduplicates nvidia-docker2 | tail -2
+nvidia-docker2.noarch           2.5.0-1                            nvidia-docker
+nvidia-docker2.noarch           2.6.0-1                            nvidia-docker
+```
+
+Installing `nvidia-docker2` shows:
+```bash
+$ yum install -y nvidia-docker2
+```
+
+installs the following packages:
+```bash
+$ yum list installed | grep nvidia
+libnvidia-container-tools.x86_64   1.5.1-1                                 @libnvidia-container
+libnvidia-container1.x86_64        1.5.1-1                                 @libnvidia-container
+nvidia-container-runtime.x86_64    3.5.0-1                                 @nvidia-container-runtime
+nvidia-container-toolkit.x86_64    1.5.1-2                                 @nvidia-container-runtime
+nvidia-docker2.noarch              2.6.0-1                                 @nvidia-docker
+```
+Note the repositories where these packages were installed from.
+
+We now enable the `local-repository` to simulate the new packages being published to the `libnvidia-container` experimental repository and check the available `nvidia-docker2` versions:
+```bash
+$ yum-config-manager --enable local-repository
+$ yum list --showduplicates nvidia-docker2 | tail -2
+nvidia-docker2.noarch         2.6.0-1                           nvidia-docker
+nvidia-docker2.noarch         2.6.1-0.1.rc.1                    local-repository
+```
+Showing the new version available in the local repository.
+
+Running:
+```
+$ yum install nvidia-docker2
+Last metadata expiration check: 0:01:15 ago on Fri Sep 24 12:49:20 2021.
+Package nvidia-docker2-2.6.0-1.noarch is already installed.
+Dependencies resolved.
+===============================================================================================================================
+ Package                                Architecture        Version                        Repository                     Size
+===============================================================================================================================
+Upgrading:
+ libnvidia-container-tools              x86_64              1.6.0-0.1.rc.1                 local-repository               48 k
+ libnvidia-container1                   x86_64              1.6.0-0.1.rc.1                 local-repository               95 k
+ nvidia-container-toolkit               x86_64              1.6.0-0.1.rc.1                 local-repository              1.5 M
+     replacing  nvidia-container-runtime.x86_64 3.5.0-1
+ nvidia-docker2                         noarch              2.6.1-0.1.rc.1                 local-repository               13 k
+
+Transaction Summary
+===============================================================================================================================
+Upgrade  4 Packages
+
+Total size: 1.7 M
+```
+Showing that all the components of the stack will be updated with versions from the `local-repository`.
+
+After installation the installed packages are shown as:
+```bash
+$ yum list installed | grep nvidia
+libnvidia-container-tools.x86_64   1.6.0-0.1.rc.1                          @local-repository
+libnvidia-container1.x86_64        1.6.0-0.1.rc.1                          @local-repository
+nvidia-container-toolkit.x86_64    1.6.0-0.1.rc.1                          @local-repository
+nvidia-docker2.noarch              2.6.1-0.1.rc.1                          @local-repository
+```
+Showing that:
+1. All versions have been installed from the same repository
+2. The `nvidia-container-runtime` package was removed as it is no longer required.
+
+The `nvidia-container-runtime`  executable is, however, still present on the system:
+```bash
+# ls -l /usr/bin/nvidia-container-runtime
+-rwxr-xr-x 1 root root 2256280 Sep 24 12:42 /usr/bin/nvidia-container-runtime
+```

test/release/docker/centos8/Dockerfile

@@ -0,0 +1,35 @@
+FROM centos:8
+
+RUN yum install -y \
+    yum-utils \
+    ruby-devel \
+    gcc \
+    make \
+    rpm-build \
+    rubygems \
+    createrepo
+
+RUN gem install --no-document fpm
+
+# We create and install a dummy docker package since these dependencies are out of
+# scope for the tests performed here.
+RUN fpm -s empty \
+    -t rpm \
+    --description "A dummy package for docker-ce_18.06.3.ce-3.el7" \
+    -n docker-ce --version 18.06.3.ce-3.el7 \
+    -p /tmp/docker.rpm \
+    && \
+    yum localinstall -y /tmp/docker.rpm \
+    && \
+    rm -f /tmp/docker.rpm
+
+
+ARG WORKFLOW=nvidia-docker
+ARG TEST_REPO=nvidia.github.io
+ENV TEST_REPO ${TEST_REPO}
+RUN curl -s -L https://nvidia.github.io/${WORKFLOW}/centos8/nvidia-docker.repo \
+    | tee /etc/yum.repos.d/nvidia-docker.repo
+
+COPY entrypoint.sh /
+
+ENTRYPOINT [ "/entrypoint.sh" ]

test/release/docker/centos8/entrypoint.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to build the packages for the components of the NVIDIA
+# Container Stack. These include the nvidia-container-toolkit in this repository
+# as well as the components included in the third_party folder.
+# All required packages are generated in the specified dist folder.
+
+: ${LOCAL_REPO_DIRECTORY:=/local-repository}
+if [[ -d ${LOCAL_REPO_DIRECTORY} ]]; then
+    echo "Setting up local-repository"
+    createrepo /local-repository
+
+    cat >/etc/yum.repos.d/local.repo <<EOL
+[local-repository]
+name=NVIDIA Container Toolkit Local Packages
+baseurl=file:///local-repository
+enabled=0
+gpgcheck=0
+protect=1
+EOL
+    yum-config-manager --enable local-repository
+else
+    echo "Setting up TEST repo: ${TEST_REPO}"
+    sed -i -e 's#nvidia\.github\.io/libnvidia-container#${TEST_REPO}/libnvidia-container#g' /etc/yum.repos.d/nvidia-docker.repo
+    yum-config-manager --enable libnvidia-container-experimental
+fi
+
+exec bash $@

test/release/docker/ubuntu18.04/Dockerfile

@@ -0,0 +1,50 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM ubuntu:18.04
+
+ARG DEBIAN_FRONTEND noninteractive
+RUN apt-get update && apt-get install --no-install-recommends -y \
+    curl \
+    gnupg2 \
+    apt-transport-https \
+    ca-certificates \
+    apt-utils \
+    ruby ruby-dev rubygems build-essential
+
+RUN gem install --no-document fpm
+
+# We create and install a dummy docker package since these dependencies are out of
+# scope for the tests performed here.
+RUN fpm -s empty \
+    -t deb \
+    --description "A dummy package for docker.io_18.06.0" \
+    -n docker.io --version 18.06.0 \
+    -p /tmp/docker.deb \
+    --deb-no-default-config-files \
+    && \
+    dpkg -i /tmp/docker.deb \
+    && \
+    rm -f /tmp/docker.deb
+
+
+ARG WORKFLOW=nvidia-docker
+RUN curl -s -L https://nvidia.github.io/${WORKFLOW}/gpgkey | apt-key add - \
+   && curl -s -L https://nvidia.github.io/${WORKFLOW}/ubuntu18.04/nvidia-docker.list | tee /etc/apt/sources.list.d/nvidia-docker.list \
+   && apt-get update
+
+COPY entrypoint.sh /
+COPY install_repo.sh /
+
+ENTRYPOINT [ "/entrypoint.sh" ]

test/release/docker/ubuntu18.04/entrypoint.sh

@@ -0,0 +1,34 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to build the packages for the components of the NVIDIA
+# Container Stack. These include the nvidia-container-toolkit in this repository
+# as well as the components included in the third_party folder.
+# All required packages are generated in the specified dist folder.
+
+: ${LOCAL_REPO_DIRECTORY:=/local-repository}
+if [[ -d ${LOCAL_REPO_DIRECTORY} ]]; then
+    echo "Setting up local-repository"
+    echo "deb [trusted=yes] file:/local-repository ./" > /etc/apt/sources.list.d/local.list
+    $(cd /local-repository && apt-ftparchive packages . > Packages)
+elif [[ -n ${TEST_REPO} ]]; then
+    ./install_repo.sh ${TEST_REPO}
+else
+    echo "Skipping repo setup"
+fi
+apt-get update
+
+exec bash $@

test/release/docker/ubuntu18.04/install_repo.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This script is used to build the packages for the components of the NVIDIA
+# Container Stack. These include the nvidia-container-toolkit in this repository
+# as well as the components included in the third_party folder.
+# All required packages are generated in the specified dist folder.
+
+test_repo=$1
+echo "Setting up TEST repo: ${test_repo}"
+sed -i -e "s#nvidia\.github\.io/libnvidia-container#${test_repo}/libnvidia-container#g" /etc/apt/sources.list.d/nvidia-docker.list
+sed -i -e '/experimental/ s/^#//g' /etc/apt/sources.list.d/nvidia-docker.list

tools/container/README.md

@@ -0,0 +1,77 @@
+## Introduction
+
+This repository contains tools that allow docker, containerd, or cri-o to be configured to use the NVIDIA Container Toolkit.
+
+*Note*: These were copied from the [`container-config` repository](https://gitlab.com/nvidia/container-toolkit/container-config/-/tree/383587f766a55177ede0e39e3810a974043e503e) are being migrated to commands installed with the NVIDIA Container Toolkit.
+
+These will be migrated into an upcoming `nvidia-ctk` CLI as required.
+
+### Docker
+
+After building the `docker` binary, run:
+```bash
+docker setup \
+    --runtime-name NAME \
+        /run/nvidia/toolkit
+```
+
+Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
+
+Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`.
+
+**Note**: If `--runtime-name` is specified as `nvidia-experimental` explicitly, the `nvidia-experimental` runtime will be configured as the default runtime, with the `nvidia` runtime still configured and available for use.
+
+The following table describes the behaviour for different `--runtime-name` and `--set-as-default` flag combinations.
+
+| Flags                                                       | Installed Runtimes              | Default Runtime       |
+|-------------------------------------------------------------|:--------------------------------|:----------------------|
+| **NONE SPECIFIED**                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--runtime-name nvidia`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--runtime-name NAME`                                       | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--runtime-name nvidia-experimental`                        | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| `--set-as-default`                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-name nvidia`                    | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-name NAME`                      | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--set-as-default --runtime-name nvidia-experimental`       | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| `--set-as-default=false`                                    | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default=false --runtime-name NAME`                | `NAME`, `nvidia-experimental`   | **NOT SET**           |
+| `--set-as-default=false --runtime-name nvidia`              | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default=false --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+
+These combinations also hold for the environment variables that map to the command line flags: `DOCKER_RUNTIME_NAME`, `DOCKER_SET_AS_DEFAULT`.
+
+### Containerd
+After running the `containerd` binary, run:
+```bash
+containerd setup \
+    --runtime-class NAME \
+        /run/nvidia/toolkit
+```
+
+Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
+
+Adding the `--set-as-default` flag as follows:
+```bash
+containerd setup \
+    --runtime-class NAME \
+    --set-as-default \
+        /run/nvidia/toolkit
+```
+will set the runtime class `NAME` (or `nvidia` if not specified) as the default runtime class.
+
+**Note**: If `--runtime-class` is specified as `nvidia-experimental` explicitly and `--set-as-default` is specified, the `nvidia-experimental` runtime will be configured as the default runtime class, with the `nvidia` runtime class still configured and available for use.
+
+The following table describes the behaviour for different `--runtime-class` and `--set-as-default` flag combinations.
+
+| Flags                                                  | Installed Runtime Classes       | Default Runtime Class |
+|--------------------------------------------------------|:--------------------------------|:----------------------|
+| **NONE SPECIFIED**                                     | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--runtime-class NAME`                                 | `NAME`, `nvidia-experimental`   | **NOT SET**           |
+| `--runtime-class nvidia`                               | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--runtime-class nvidia-experimental`                  | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-class NAME`                | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--set-as-default --runtime-class nvidia`              | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+
+These combinations also hold for the environment variables that map to the command line flags.

tools/container/containerd/config.go

@@ -0,0 +1,116 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/pelletier/go-toml"
+)
+
+// UpdateReverter defines the interface for applying and reverting configurations
+type UpdateReverter interface {
+	Update(o *options) error
+	Revert(o *options) error
+}
+
+type config struct {
+	*toml.Tree
+	version   int64
+	cri       string
+	binaryKey string
+}
+
+// update adds the specified runtime class to the the containerd config.
+// if set-as default is specified, the runtime class is also set as the
+// default runtime.
+func (config *config) update(runtimeClass string, runtimeType string, runtimeBinary string, setAsDefault bool) {
+	config.Set("version", config.version)
+
+	runcPath := config.runcPath()
+	runtimeClassPath := config.runtimeClassPath(runtimeClass)
+
+	switch runc := config.GetPath(runcPath).(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath(runtimeClassPath, runc)
+	}
+
+	config.initRuntime(runtimeClassPath, runtimeType, runtimeBinary)
+
+	if setAsDefault {
+		defaultRuntimeNamePath := config.defaultRuntimeNamePath()
+		config.SetPath(defaultRuntimeNamePath, runtimeClass)
+	}
+}
+
+// revert removes the configuration applied in an update call.
+func (config *config) revert(runtimeClass string) {
+	runtimeClassPath := config.runtimeClassPath(runtimeClass)
+	defaultRuntimeNamePath := config.defaultRuntimeNamePath()
+
+	config.DeletePath(runtimeClassPath)
+	if runtime, ok := config.GetPath(defaultRuntimeNamePath).(string); ok {
+		if runtimeClass == runtime {
+			config.DeletePath(defaultRuntimeNamePath)
+		}
+	}
+
+	for i := 0; i < len(runtimeClassPath); i++ {
+		if runtimes, ok := config.GetPath(runtimeClassPath[:len(runtimeClassPath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimeClassPath[:len(runtimeClassPath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+}
+
+// initRuntime creates a runtime config if it does not exist and ensures that the
+// runtimes binary path is specified.
+func (config *config) initRuntime(path []string, runtimeType string, binary string) {
+	if config.GetPath(path) == nil {
+		config.SetPath(append(path, "runtime_type"), runtimeType)
+		config.SetPath(append(path, "runtime_root"), "")
+		config.SetPath(append(path, "runtime_engine"), "")
+		config.SetPath(append(path, "privileged_without_host_devices"), false)
+	}
+
+	binaryPath := append(path, "options", config.binaryKey)
+	config.SetPath(binaryPath, binary)
+}
+
+func (config config) runcPath() []string {
+	return config.runtimeClassPath("runc")
+}
+
+func (config config) runtimeClassBinaryPath(runtimeClass string) []string {
+	return append(config.runtimeClassPath(runtimeClass), "options", config.binaryKey)
+}
+
+func (config config) runtimeClassPath(runtimeClass string) []string {
+	return append(config.containerdPath(), "runtimes", runtimeClass)
+}
+
+func (config config) defaultRuntimeNamePath() []string {
+	return append(config.containerdPath(), "default_runtime_name")
+}
+
+func (config config) containerdPath() []string {
+	return []string{"plugins", config.cri, "containerd"}
+}

tools/container/containerd/config_v1.go

@@ -0,0 +1,126 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"path"
+
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+// configV1 represents a V1 containerd config
+type configV1 struct {
+	config
+}
+
+func newConfigV1(cfg *toml.Tree) UpdateReverter {
+	c := configV1{
+		config: config{
+			Tree:      cfg,
+			version:   1,
+			cri:       "cri",
+			binaryKey: "Runtime",
+		},
+	}
+
+	return &c
+}
+
+// Update performs an update specific to v1 of the containerd config
+func (config *configV1) Update(o *options) error {
+
+	// For v1 config, the `default_runtime_name` setting is only supported
+	// for containerd version at least v1.3
+	supportsDefaultRuntimeName := !o.useLegacyConfig
+
+	defaultRuntime := o.getDefaultRuntime()
+
+	for runtimeClass, runtimeBinary := range o.getRuntimeBinaries() {
+		isDefaultRuntime := runtimeClass == defaultRuntime
+		config.update(runtimeClass, o.runtimeType, runtimeBinary, isDefaultRuntime && supportsDefaultRuntimeName)
+
+		if !isDefaultRuntime {
+			continue
+		}
+
+		if supportsDefaultRuntimeName {
+			defaultRuntimePath := append(config.containerdPath(), "default_runtime")
+			if config.GetPath(defaultRuntimePath) != nil {
+				log.Warnf("The setting of default_runtime (%v) in containerd is deprecated", defaultRuntimePath)
+			}
+			continue
+		}
+
+		log.Warnf("Setting default_runtime is deprecated")
+		defaultRuntimePath := append(config.containerdPath(), "default_runtime")
+		config.initRuntime(defaultRuntimePath, o.runtimeType, runtimeBinary)
+	}
+	return nil
+}
+
+// Revert performs a revert specific to v1 of the containerd config
+func (config *configV1) Revert(o *options) error {
+	defaultRuntimePath := append(config.containerdPath(), "default_runtime")
+	defaultRuntimeOptionsPath := append(defaultRuntimePath, "options")
+	if runtime, ok := config.GetPath(append(defaultRuntimeOptionsPath, "Runtime")).(string); ok {
+		for _, runtimeBinary := range o.getRuntimeBinaries() {
+			if path.Base(runtimeBinary) == path.Base(runtime) {
+				config.DeletePath(append(defaultRuntimeOptionsPath, "Runtime"))
+				break
+			}
+		}
+	}
+
+	if options, ok := config.GetPath(defaultRuntimeOptionsPath).(*toml.Tree); ok {
+		if len(options.Keys()) == 0 {
+			config.DeletePath(defaultRuntimeOptionsPath)
+		}
+	}
+
+	if runtime, ok := config.GetPath(defaultRuntimePath).(*toml.Tree); ok {
+		fields := []string{"runtime_type", "runtime_root", "runtime_engine", "privileged_without_host_devices"}
+		if len(runtime.Keys()) <= len(fields) {
+			matches := []string{}
+			for _, f := range fields {
+				e := runtime.Get(f)
+				if e != nil {
+					matches = append(matches, f)
+				}
+			}
+			if len(matches) == len(runtime.Keys()) {
+				for _, m := range matches {
+					runtime.Delete(m)
+				}
+			}
+		}
+	}
+
+	for i := 0; i < len(defaultRuntimePath); i++ {
+		if runtimes, ok := config.GetPath(defaultRuntimePath[:len(defaultRuntimePath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(defaultRuntimePath[:len(defaultRuntimePath)-i])
+			}
+		}
+	}
+
+	for runtimeClass := range nvidiaRuntimeBinaries {
+		config.revert(runtimeClass)
+	}
+
+	return nil
+}

tools/container/containerd/config_v1_test.go

@@ -0,0 +1,365 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/pelletier/go-toml"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		legacyConfig                 bool
+		setAsDefault                 bool
+		runtimeClass                 string
+		expectedDefaultRuntimeName   interface{}
+		expectedDefaultRuntimeBinary interface{}
+	}{
+		{},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 false,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			runtimeClass:                 "NAME",
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			runtimeClass:                 "nvidia-experimental",
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime-experimental",
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 false,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			expectedDefaultRuntimeName:   "nvidia",
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			runtimeClass:                 "NAME",
+			expectedDefaultRuntimeName:   "NAME",
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			runtimeClass:                 "nvidia-experimental",
+			expectedDefaultRuntimeName:   "nvidia-experimental",
+			expectedDefaultRuntimeBinary: nil,
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			useLegacyConfig: tc.legacyConfig,
+			setAsDefault:    tc.setAsDefault,
+			runtimeClass:    tc.runtimeClass,
+			runtimeType:     runtimeType,
+			runtimeDir:      runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(map[string]interface{}{})
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV1Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		defaultRuntimeName := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
+		require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
+
+		defaultRuntime := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"})
+		if tc.expectedDefaultRuntimeBinary == nil {
+			require.Nil(t, defaultRuntime, "%d: %v", i, tc)
+		} else {
+			expected, err := runtimeTomlConfigV1(tc.expectedDefaultRuntimeBinary.(string))
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(defaultRuntime.(*toml.Tree))
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, tc)
+		}
+
+	}
+}
+
+func TestUpdateV1Config(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+	const expectedVersion = int64(1)
+
+	expectedBinaries := []string{
+		"/test/runtime/dir/nvidia-container-runtime",
+		"/test/runtime/dir/nvidia-container-runtime-experimental",
+	}
+
+	testCases := []struct {
+		runtimeClass     string
+		expectedRuntimes []string
+	}{
+		{
+			runtimeClass:     "nvidia",
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "NAME",
+			expectedRuntimes: []string{"NAME", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "nvidia-experimental",
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: tc.runtimeClass,
+			runtimeType:  runtimeType,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(map[string]interface{}{})
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV1Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		version, ok := config.Get("version").(int64)
+		require.True(t, ok)
+		require.EqualValues(t, expectedVersion, version)
+
+		runtimes, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes"}).(*toml.Tree)
+		require.True(t, ok)
+
+		runtimeClasses := runtimes.Keys()
+		require.ElementsMatch(t, tc.expectedRuntimes, runtimeClasses, "%d: %v", i, tc)
+
+		for i, r := range tc.expectedRuntimes {
+			runtimeConfig := runtimes.Get(r)
+
+			expected, err := runtimeTomlConfigV1(expectedBinaries[i])
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(runtimeConfig)
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
+
+		}
+	}
+}
+
+func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
+	const runcBinary = "/runc-binary"
+	const runtimeDir = "/test/runtime/dir"
+	const expectedVersion = int64(1)
+
+	expectedBinaries := []string{
+		runcBinary,
+		"/test/runtime/dir/nvidia-container-runtime",
+		"/test/runtime/dir/nvidia-container-runtime-experimental",
+	}
+
+	testCases := []struct {
+		runtimeClass     string
+		expectedRuntimes []string
+	}{
+		{
+			runtimeClass:     "nvidia",
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "NAME",
+			expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "nvidia-experimental",
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: tc.runtimeClass,
+			runtimeType:  runtimeType,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(runcConfigMapV1("/runc-binary"))
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV1Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		version, ok := config.Get("version").(int64)
+		require.True(t, ok)
+		require.EqualValues(t, expectedVersion, version)
+
+		runtimes, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes"}).(*toml.Tree)
+		require.True(t, ok)
+
+		runtimeClasses := runtimes.Keys()
+		require.ElementsMatch(t, tc.expectedRuntimes, runtimeClasses, "%d: %v", i, tc)
+
+		for i, r := range tc.expectedRuntimes {
+			runtimeConfig := runtimes.Get(r)
+
+			expected, err := toml.TreeFromMap(runcRuntimeConfigMapV1(expectedBinaries[i]))
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(runtimeConfig)
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
+
+		}
+	}
+}
+
+func TestRevertV1Config(t *testing.T) {
+	testCases := []struct {
+		config map[string]interface {
+		}
+		expected map[string]interface{}
+	}{
+		{},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":              runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime-experimental"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":              runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime-experimental"),
+							},
+							"default_runtime":      runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+							"default_runtime_name": "nvidia",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: "nvidia",
+		}
+
+		config, err := toml.TreeFromMap(tc.config)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		expected, err := toml.TreeFromMap(tc.expected)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = RevertV1Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		configContents, _ := toml.Marshal(config)
+		expectedContents, _ := toml.Marshal(expected)
+
+		require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
+	}
+}
+
+func runtimeTomlConfigV1(binary string) (*toml.Tree, error) {
+	return toml.TreeFromMap(runtimeMapV1(binary))
+}
+
+func runtimeMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"Runtime": binary,
+		},
+	}
+}
+
+func runcConfigMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"plugins": map[string]interface{}{
+			"cri": map[string]interface{}{
+				"containerd": map[string]interface{}{
+					"runtimes": map[string]interface{}{
+						"runc": runcRuntimeConfigMapV1(binary),
+					},
+				},
+			},
+		},
+	}
+}
+
+func runcRuntimeConfigMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    "runc_runtime_type",
+		"runtime_root":                    "runc_runtime_root",
+		"runtime_engine":                  "runc_runtime_engine",
+		"privileged_without_host_devices": true,
+		"options": map[string]interface{}{
+			"runc-option": "value",
+			"Runtime":     binary,
+		},
+	}
+}

tools/container/containerd/config_v2.go

@@ -0,0 +1,59 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/pelletier/go-toml"
+)
+
+// configV2 represents a V2 containerd config
+type configV2 struct {
+	config
+}
+
+func newConfigV2(cfg *toml.Tree) UpdateReverter {
+	c := configV2{
+		config: config{
+			Tree:      cfg,
+			version:   2,
+			cri:       "io.containerd.grpc.v1.cri",
+			binaryKey: "BinaryName",
+		},
+	}
+
+	return &c
+}
+
+// Update performs an update specific to v2 of the containerd config
+func (config *configV2) Update(o *options) error {
+	defaultRuntime := o.getDefaultRuntime()
+	for runtimeClass, runtimeBinary := range o.getRuntimeBinaries() {
+		setAsDefault := defaultRuntime == runtimeClass
+		config.update(runtimeClass, o.runtimeType, runtimeBinary, setAsDefault)
+	}
+
+	return nil
+}
+
+// Revert performs a revert specific to v2 of the containerd config
+func (config *configV2) Revert(o *options) error {
+	for runtimeClass := range o.getRuntimeBinaries() {
+		config.revert(runtimeClass)
+	}
+
+	return nil
+}

tools/container/containerd/config_v2_test.go

@@ -0,0 +1,329 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/pelletier/go-toml"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	runtimeType = "runtime_type"
+)
+
+func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		setAsDefault               bool
+		runtimeClass               string
+		expectedDefaultRuntimeName interface{}
+	}{
+		{},
+		{
+			setAsDefault:               false,
+			runtimeClass:               "nvidia",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               false,
+			runtimeClass:               "NAME",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               false,
+			runtimeClass:               "nvidia-experimental",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               true,
+			runtimeClass:               "nvidia",
+			expectedDefaultRuntimeName: "nvidia",
+		},
+		{
+			setAsDefault:               true,
+			runtimeClass:               "NAME",
+			expectedDefaultRuntimeName: "NAME",
+		},
+		{
+			setAsDefault:               true,
+			runtimeClass:               "nvidia-experimental",
+			expectedDefaultRuntimeName: "nvidia-experimental",
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			setAsDefault: tc.setAsDefault,
+			runtimeClass: tc.runtimeClass,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(map[string]interface{}{})
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV2Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+		require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
+	}
+}
+
+func TestUpdateV2Config(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+	const expectedVersion = int64(2)
+
+	expectedBinaries := []string{
+		"/test/runtime/dir/nvidia-container-runtime",
+		"/test/runtime/dir/nvidia-container-runtime-experimental",
+	}
+
+	testCases := []struct {
+		runtimeClass     string
+		expectedRuntimes []string
+	}{
+		{
+			runtimeClass:     "nvidia",
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "NAME",
+			expectedRuntimes: []string{"NAME", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "nvidia-experimental",
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: tc.runtimeClass,
+			runtimeType:  runtimeType,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(map[string]interface{}{})
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV2Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		version, ok := config.Get("version").(int64)
+		require.True(t, ok)
+		require.EqualValues(t, expectedVersion, version, "%d: %v", i, tc)
+
+		runtimes, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes"}).(*toml.Tree)
+		require.True(t, ok)
+
+		runtimeClasses := runtimes.Keys()
+		require.ElementsMatch(t, tc.expectedRuntimes, runtimeClasses, "%d: %v", i, tc)
+
+		for i, r := range tc.expectedRuntimes {
+			runtimeConfig := runtimes.Get(r)
+
+			expected, err := runtimeTomlConfigV2(expectedBinaries[i])
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(runtimeConfig)
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
+
+		}
+	}
+
+}
+
+func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
+	const runcBinary = "/runc-binary"
+	const runtimeDir = "/test/runtime/dir"
+	const expectedVersion = int64(2)
+
+	expectedBinaries := []string{
+		runcBinary,
+		"/test/runtime/dir/nvidia-container-runtime",
+		"/test/runtime/dir/nvidia-container-runtime-experimental",
+	}
+
+	testCases := []struct {
+		runtimeClass     string
+		expectedRuntimes []string
+	}{
+		{
+			runtimeClass:     "nvidia",
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "NAME",
+			expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "nvidia-experimental",
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: tc.runtimeClass,
+			runtimeType:  runtimeType,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(runcConfigMapV2("/runc-binary"))
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV2Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		version, ok := config.Get("version").(int64)
+		require.True(t, ok)
+		require.EqualValues(t, expectedVersion, version)
+
+		runtimes, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes"}).(*toml.Tree)
+		require.True(t, ok, "%d: %v", i, tc)
+
+		runtimeClasses := runtimes.Keys()
+		require.ElementsMatch(t, tc.expectedRuntimes, runtimeClasses, "%d: %v", i, tc)
+
+		for i, r := range tc.expectedRuntimes {
+			runtimeConfig := runtimes.Get(r)
+
+			expected, err := toml.TreeFromMap(runcRuntimeConfigMapV2(expectedBinaries[i]))
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(runtimeConfig)
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
+
+		}
+	}
+}
+
+func TestRevertV2Config(t *testing.T) {
+	testCases := []struct {
+		config map[string]interface {
+		}
+		expected map[string]interface{}
+	}{
+		{},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":              runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime-experimental"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":              runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime-experimental"),
+							},
+							"default_runtime_name": "nvidia",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: "nvidia",
+		}
+
+		config, err := toml.TreeFromMap(tc.config)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		expected, err := toml.TreeFromMap(tc.expected)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = RevertV2Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		configContents, _ := toml.Marshal(config)
+		expectedContents, _ := toml.Marshal(expected)
+
+		require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
+	}
+}
+
+func runtimeTomlConfigV2(binary string) (*toml.Tree, error) {
+	return toml.TreeFromMap(runtimeMapV2(binary))
+}
+
+func runtimeMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"BinaryName": binary,
+		},
+	}
+}
+
+func runcConfigMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"plugins": map[string]interface{}{
+			"io.containerd.grpc.v1.cri": map[string]interface{}{
+				"containerd": map[string]interface{}{
+					"runtimes": map[string]interface{}{
+						"runc": runcRuntimeConfigMapV2(binary),
+					},
+				},
+			},
+		},
+	}
+}
+
+func runcRuntimeConfigMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    "runc_runtime_type",
+		"runtime_root":                    "runc_runtime_root",
+		"runtime_engine":                  "runc_runtime_engine",
+		"privileged_without_host_devices": true,
+		"options": map[string]interface{}{
+			"runc-option": "value",
+			"BinaryName":  binary,
+		},
+	}
+}

tools/container/containerd/containerd.go

@@ -0,0 +1,586 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	toml "github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+)
+
+const (
+	restartModeSignal  = "signal"
+	restartModeSystemd = "systemd"
+	restartModeNone    = "NONE"
+
+	nvidiaRuntimeName               = "nvidia"
+	nvidiaRuntimeBinary             = "nvidia-container-runtime"
+	nvidiaExperimentalRuntimeName   = "nvidia-experimental"
+	nvidiaExperimentalRuntimeBinary = "nvidia-container-runtime-experimental"
+
+	defaultConfig        = "/etc/containerd/config.toml"
+	defaultSocket        = "/run/containerd/containerd.sock"
+	defaultRuntimeClass  = "nvidia"
+	defaultRuntmeType    = "io.containerd.runc.v2"
+	defaultSetAsDefault  = true
+	defaultRestartMode   = restartModeSignal
+	defaultHostRootMount = "/host"
+
+	reloadBackoff     = 5 * time.Second
+	maxReloadAttempts = 6
+
+	socketMessageToGetPID = ""
+)
+
+// nvidiaRuntimeBinaries defines a map of runtime names to binary names
+var nvidiaRuntimeBinaries = map[string]string{
+	nvidiaRuntimeName:             nvidiaRuntimeBinary,
+	nvidiaExperimentalRuntimeName: nvidiaExperimentalRuntimeBinary,
+}
+
+// options stores the configuration from the command line or environment variables
+type options struct {
+	config          string
+	socket          string
+	runtimeClass    string
+	runtimeType     string
+	setAsDefault    bool
+	restartMode     string
+	hostRootMount   string
+	runtimeDir      string
+	useLegacyConfig bool
+}
+
+func main() {
+	options := options{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "containerd"
+	c.Usage = "Update a containerd config with the nvidia-container-runtime"
+	c.Version = "0.1.0"
+
+	// Create the 'setup' subcommand
+	setup := cli.Command{}
+	setup.Name = "setup"
+	setup.Usage = "Trigger a containerd config to be updated"
+	setup.ArgsUsage = "<runtime_dirname>"
+	setup.Action = func(c *cli.Context) error {
+		return Setup(c, &options)
+	}
+
+	// Create the 'cleanup' subcommand
+	cleanup := cli.Command{}
+	cleanup.Name = "cleanup"
+	cleanup.Usage = "Trigger any updates made to a containerd config to be undone"
+	cleanup.ArgsUsage = "<runtime_dirname>"
+	cleanup.Action = func(c *cli.Context) error {
+		return Cleanup(c, &options)
+	}
+
+	// Register the subcommands with the top-level CLI
+	c.Commands = []*cli.Command{
+		&setup,
+		&cleanup,
+	}
+
+	// Setup common flags across both subcommands. All subcommands get the same
+	// set of flags even if they don't use some of them. This is so that we
+	// only require the user to specify one set of flags for both 'startup'
+	// and 'cleanup' to simplify things.
+	commonFlags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config",
+			Aliases:     []string{"c"},
+			Usage:       "Path to the containerd config file",
+			Value:       defaultConfig,
+			Destination: &options.config,
+			EnvVars:     []string{"CONTAINERD_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "socket",
+			Aliases:     []string{"s"},
+			Usage:       "Path to the containerd socket file",
+			Value:       defaultSocket,
+			Destination: &options.socket,
+			EnvVars:     []string{"CONTAINERD_SOCKET"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-class",
+			Aliases:     []string{"r"},
+			Usage:       "The name of the runtime class to set for the nvidia-container-runtime",
+			Value:       defaultRuntimeClass,
+			Destination: &options.runtimeClass,
+			EnvVars:     []string{"CONTAINERD_RUNTIME_CLASS"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-type",
+			Usage:       "The runtime_type to use for the configured runtime classes",
+			Value:       defaultRuntmeType,
+			Destination: &options.runtimeType,
+			EnvVars:     []string{"CONTAINERD_RUNTIME_TYPE"},
+		},
+		// The flags below are only used by the 'setup' command.
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Aliases:     []string{"d"},
+			Usage:       "Set nvidia-container-runtime as the default runtime",
+			Value:       defaultSetAsDefault,
+			Destination: &options.setAsDefault,
+			EnvVars:     []string{"CONTAINERD_SET_AS_DEFAULT"},
+			Hidden:      true,
+		},
+		&cli.StringFlag{
+			Name:        "restart-mode",
+			Usage:       "Specify how containerd should be restarted; [signal | systemd]",
+			Value:       defaultRestartMode,
+			Destination: &options.restartMode,
+			EnvVars:     []string{"CONTAINERD_RESTART_MODE"},
+		},
+		&cli.StringFlag{
+			Name:        "host-root",
+			Usage:       "Specify the path to the host root to be used when restarting containerd using systemd",
+			Value:       defaultHostRootMount,
+			Destination: &options.hostRootMount,
+			EnvVars:     []string{"HOST_ROOT_MOUNT"},
+		},
+		&cli.BoolFlag{
+			Name:        "use-legacy-config",
+			Usage:       "Specify whether a legacy (pre v1.3) config should be used",
+			Destination: &options.useLegacyConfig,
+			EnvVars:     []string{"CONTAINERD_USE_LEGACY_CONFIG"},
+		},
+	}
+
+	// Update the subcommand flags with the common subcommand flags
+	setup.Flags = append([]cli.Flag{}, commonFlags...)
+	cleanup.Flags = append([]cli.Flag{}, commonFlags...)
+
+	// Run the top-level CLI
+	if err := c.Run(os.Args); err != nil {
+		log.Fatal(fmt.Errorf("Error: %v", err))
+	}
+}
+
+// Setup updates a containerd configuration to include the nvidia-containerd-runtime and reloads it
+func Setup(c *cli.Context, o *options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	runtimeDir, err := ParseArgs(c)
+	if err != nil {
+		return fmt.Errorf("unable to parse args: %v", err)
+	}
+	o.runtimeDir = runtimeDir
+
+	cfg, err := LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	version, err := ParseVersion(cfg, o.useLegacyConfig)
+	if err != nil {
+		return fmt.Errorf("unable to parse version: %v", err)
+	}
+
+	err = UpdateConfig(cfg, o, version)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = FlushConfig(o.config, cfg)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = RestartContainerd(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart containerd: %v", err)
+	}
+
+	log.Infof("Completed 'setup' for %v", c.App.Name)
+
+	return nil
+}
+
+// Cleanup reverts a containerd configuration to remove the nvidia-containerd-runtime and reloads it
+func Cleanup(c *cli.Context, o *options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	_, err := ParseArgs(c)
+	if err != nil {
+		return fmt.Errorf("unable to parse args: %v", err)
+	}
+
+	cfg, err := LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	version, err := ParseVersion(cfg, o.useLegacyConfig)
+	if err != nil {
+		return fmt.Errorf("unable to parse version: %v", err)
+	}
+
+	err = RevertConfig(cfg, o, version)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = FlushConfig(o.config, cfg)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = RestartContainerd(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart containerd: %v", err)
+	}
+
+	log.Infof("Completed 'cleanup' for %v", c.App.Name)
+
+	return nil
+}
+
+// ParseArgs parses the command line arguments to the CLI
+func ParseArgs(c *cli.Context) (string, error) {
+	args := c.Args()
+
+	log.Infof("Parsing arguments: %v", args.Slice())
+	if args.Len() != 1 {
+		return "", fmt.Errorf("incorrect number of arguments")
+	}
+	runtimeDir := args.Get(0)
+	log.Infof("Successfully parsed arguments")
+
+	return runtimeDir, nil
+}
+
+// LoadConfig loads the containerd config from disk
+func LoadConfig(config string) (*toml.Tree, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	cfg, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	return cfg, nil
+}
+
+// ParseVersion parses the version field out of the containerd config
+func ParseVersion(config *toml.Tree, useLegacyConfig bool) (int, error) {
+	var defaultVersion int
+	if !useLegacyConfig {
+		defaultVersion = 2
+	} else {
+		defaultVersion = 1
+	}
+
+	var version int
+	switch v := config.Get("version").(type) {
+	case nil:
+		switch len(config.Keys()) {
+		case 0: // No config exists, or the config file is empty, use version inferred from containerd
+			version = defaultVersion
+		default: // A config file exists, has content, and no version is set
+			version = 1
+		}
+	case int64:
+		version = int(v)
+	default:
+		return -1, fmt.Errorf("unsupported type for version field: %v", v)
+	}
+	log.Infof("Config version: %v", version)
+
+	if version == 1 {
+		log.Warnf("Support for containerd config version 1 is deprecated")
+	}
+
+	return version, nil
+}
+
+// UpdateConfig updates the containerd config to include the nvidia-container-runtime
+func UpdateConfig(config *toml.Tree, o *options, version int) error {
+	var err error
+
+	log.Infof("Updating config")
+	switch version {
+	case 1:
+		err = UpdateV1Config(config, o)
+	case 2:
+		err = UpdateV2Config(config, o)
+	default:
+		err = fmt.Errorf("unsupported containerd config version: %v", version)
+	}
+	if err != nil {
+		return err
+	}
+	log.Infof("Successfully updated config")
+
+	return nil
+}
+
+// RevertConfig reverts the containerd config to remove the nvidia-container-runtime
+func RevertConfig(config *toml.Tree, o *options, version int) error {
+	var err error
+
+	log.Infof("Reverting config")
+	switch version {
+	case 1:
+		err = RevertV1Config(config, o)
+	case 2:
+		err = RevertV2Config(config, o)
+	default:
+		err = fmt.Errorf("unsupported containerd config version: %v", version)
+	}
+	if err != nil {
+		return err
+	}
+	log.Infof("Successfully reverted config")
+
+	return nil
+}
+
+// UpdateV1Config performs an update specific to v1 of the containerd config
+func UpdateV1Config(config *toml.Tree, o *options) error {
+	c := newConfigV1(config)
+	return c.Update(o)
+}
+
+// RevertV1Config performs a revert specific to v1 of the containerd config
+func RevertV1Config(config *toml.Tree, o *options) error {
+	c := newConfigV1(config)
+	return c.Revert(o)
+}
+
+// UpdateV2Config performs an update specific to v2 of the containerd config
+func UpdateV2Config(config *toml.Tree, o *options) error {
+	c := newConfigV2(config)
+	return c.Update(o)
+}
+
+// RevertV2Config performs a revert specific to v2 of the containerd config
+func RevertV2Config(config *toml.Tree, o *options) error {
+	c := newConfigV2(config)
+	return c.Revert(o)
+}
+
+// FlushConfig flushes the updated/reverted config out to disk
+func FlushConfig(config string, cfg *toml.Tree) error {
+	log.Infof("Flushing config")
+
+	output, err := cfg.ToTomlString()
+	if err != nil {
+		return fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	switch len(output) {
+	case 0:
+		err := os.Remove(config)
+		if err != nil {
+			return fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		log.Infof("Config empty, removing file")
+	default:
+		f, err := os.Create(config)
+		if err != nil {
+			return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(output)
+		if err != nil {
+			return fmt.Errorf("unable to write output: %v", err)
+		}
+	}
+
+	log.Infof("Successfully flushed config")
+
+	return nil
+}
+
+// RestartContainerd restarts containerd depending on the value of restartModeFlag
+func RestartContainerd(o *options) error {
+	switch o.restartMode {
+	case restartModeNone:
+		log.Warnf("Skipping sending signal to containerd due to --restart-mode=%v", o.restartMode)
+		return nil
+	case restartModeSignal:
+		err := SignalContainerd(o)
+		if err != nil {
+			return fmt.Errorf("unable to signal containerd: %v", err)
+		}
+	case restartModeSystemd:
+		return RestartContainerdSystemd(o.hostRootMount)
+	default:
+		return fmt.Errorf("Invalid restart mode specified: %v", o.restartMode)
+	}
+
+	return nil
+}
+
+// SignalContainerd sends a SIGHUP signal to the containerd daemon
+func SignalContainerd(o *options) error {
+	log.Infof("Sending SIGHUP signal to containerd")
+
+	// Wrap the logic to perform the SIGHUP in a function so we can retry it on failure
+	retriable := func() error {
+		conn, err := net.Dial("unix", o.socket)
+		if err != nil {
+			return fmt.Errorf("unable to dial: %v", err)
+		}
+		defer conn.Close()
+
+		sconn, err := conn.(*net.UnixConn).SyscallConn()
+		if err != nil {
+			return fmt.Errorf("unable to get syscall connection: %v", err)
+		}
+
+		err1 := sconn.Control(func(fd uintptr) {
+			err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
+		})
+		if err1 != nil {
+			return fmt.Errorf("unable to issue call on socket fd: %v", err1)
+		}
+		if err != nil {
+			return fmt.Errorf("unable to SetsockoptInt on socket fd: %v", err)
+		}
+
+		_, _, err = conn.(*net.UnixConn).WriteMsgUnix([]byte(socketMessageToGetPID), nil, nil)
+		if err != nil {
+			return fmt.Errorf("unable to WriteMsgUnix on socket fd: %v", err)
+		}
+
+		oob := make([]byte, 1024)
+		_, oobn, _, _, err := conn.(*net.UnixConn).ReadMsgUnix(nil, oob)
+		if err != nil {
+			return fmt.Errorf("unable to ReadMsgUnix on socket fd: %v", err)
+		}
+
+		oob = oob[:oobn]
+		scm, err := syscall.ParseSocketControlMessage(oob)
+		if err != nil {
+			return fmt.Errorf("unable to ParseSocketControlMessage from message received on socket fd: %v", err)
+		}
+
+		ucred, err := syscall.ParseUnixCredentials(&scm[0])
+		if err != nil {
+			return fmt.Errorf("unable to ParseUnixCredentials from message received on socket fd: %v", err)
+		}
+
+		err = syscall.Kill(int(ucred.Pid), syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("unable to send SIGHUP to 'containerd' process: %v", err)
+		}
+
+		return nil
+	}
+
+	// Try to send a SIGHUP up to maxReloadAttempts times
+	var err error
+	for i := 0; i < maxReloadAttempts; i++ {
+		err = retriable()
+		if err == nil {
+			break
+		}
+		if i == maxReloadAttempts-1 {
+			break
+		}
+		log.Warnf("Error signaling containerd, attempt %v/%v: %v", i+1, maxReloadAttempts, err)
+		time.Sleep(reloadBackoff)
+	}
+	if err != nil {
+		log.Warnf("Max retries reached %v/%v, aborting", maxReloadAttempts, maxReloadAttempts)
+		return err
+	}
+
+	log.Infof("Successfully signaled containerd")
+
+	return nil
+}
+
+// RestartContainerdSystemd restarts containerd using systemctl
+func RestartContainerdSystemd(hostRootMount string) error {
+	log.Infof("Restarting containerd using systemd and host root mounted at %v", hostRootMount)
+
+	command := "chroot"
+	args := []string{hostRootMount, "systemctl", "restart", "containerd"}
+
+	cmd := exec.Command(command, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error restarting containerd using systemd: %v", err)
+	}
+
+	return nil
+}
+
+// getDefaultRuntime returns the default runtime for the configured options.
+// If the configuration is invalid or the default runtimes should not be set
+// the empty string is returned.
+func (o options) getDefaultRuntime() string {
+	if o.setAsDefault {
+		if o.runtimeClass == nvidiaExperimentalRuntimeName {
+			return nvidiaExperimentalRuntimeName
+		}
+		if o.runtimeClass == "" {
+			return defaultRuntimeClass
+		}
+		return o.runtimeClass
+	}
+	return ""
+}
+
+// getRuntimeBinaries returns a map of runtime names to binary paths. This includes the
+// renaming of the `nvidia` runtime as per the --runtime-class command line flag.
+func (o options) getRuntimeBinaries() map[string]string {
+	runtimeBinaries := make(map[string]string)
+
+	for rt, bin := range nvidiaRuntimeBinaries {
+		runtime := rt
+		if o.runtimeClass != "" && o.runtimeClass != nvidiaExperimentalRuntimeName && runtime == defaultRuntimeClass {
+			runtime = o.runtimeClass
+		}
+
+		runtimeBinaries[runtime] = filepath.Join(o.runtimeDir, bin)
+	}
+
+	return runtimeBinaries
+}

tools/container/containerd/containerd_test.go

@@ -0,0 +1,106 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptions(t *testing.T) {
+	testCases := []struct {
+		options                 options
+		expectedDefaultRuntime  string
+		expectedRuntimeBinaries map[string]string
+	}{
+		{
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: true,
+			},
+			expectedDefaultRuntime: "nvidia",
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: true,
+				runtimeClass: "nvidia",
+			},
+			expectedDefaultRuntime: "nvidia",
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: true,
+				runtimeClass: "NAME",
+			},
+			expectedDefaultRuntime: "NAME",
+			expectedRuntimeBinaries: map[string]string{
+				"NAME":                "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: false,
+				runtimeClass: "NAME",
+			},
+			expectedRuntimeBinaries: map[string]string{
+				"NAME":                "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: true,
+				runtimeClass: "nvidia-experimental",
+			},
+			expectedDefaultRuntime: "nvidia-experimental",
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: false,
+				runtimeClass: "nvidia-experimental",
+			},
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		require.Equal(t, tc.expectedDefaultRuntime, tc.options.getDefaultRuntime(), "%d: %v", i, tc)
+		require.EqualValues(t, tc.expectedRuntimeBinaries, tc.options.getRuntimeBinaries(), "%d: %v", i, tc)
+	}
+}

tools/container/crio/crio.go

@@ -0,0 +1,185 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	hooks "github.com/containers/podman/v2/pkg/hooks/1.0.0"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+)
+
+const (
+	defaultHooksDir     = "/usr/share/containers/oci/hooks.d"
+	defaultHookFilename = "oci-nvidia-hook.json"
+)
+
+var hooksDirFlag string
+var hookFilenameFlag string
+var tooklitDirArg string
+
+func main() {
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "crio"
+	c.Usage = "Update cri-o hooks to include the NVIDIA runtime hook"
+	c.ArgsUsage = "<toolkit_dirname>"
+	c.Version = "0.1.0"
+
+	// Create the 'setup' subcommand
+	setup := cli.Command{}
+	setup.Name = "setup"
+	setup.Usage = "Create the cri-o hook required to run NVIDIA GPU containers"
+	setup.ArgsUsage = "<toolkit_dirname>"
+	setup.Action = Setup
+	setup.Before = ParseArgs
+
+	// Create the 'cleanup' subcommand
+	cleanup := cli.Command{}
+	cleanup.Name = "cleanup"
+	cleanup.Usage = "Remove the NVIDIA cri-o hook"
+	cleanup.Action = Cleanup
+
+	// Register the subcommands with the top-level CLI
+	c.Commands = []*cli.Command{
+		&setup,
+		&cleanup,
+	}
+
+	// Setup common flags across both subcommands. All subcommands get the same
+	// set of flags even if they don't use some of them. This is so that we
+	// only require the user to specify one set of flags for both 'startup'
+	// and 'cleanup' to simplify things.
+	commonFlags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "hooks-dir",
+			Aliases:     []string{"d"},
+			Usage:       "path to the cri-o hooks directory",
+			Value:       defaultHooksDir,
+			Destination: &hooksDirFlag,
+			EnvVars:     []string{"CRIO_HOOKS_DIR"},
+			DefaultText: defaultHooksDir,
+		},
+		&cli.StringFlag{
+			Name:        "hook-filename",
+			Aliases:     []string{"f"},
+			Usage:       "filename of the cri-o hook that will be created / removed in the hooks directory",
+			Value:       defaultHookFilename,
+			Destination: &hookFilenameFlag,
+			EnvVars:     []string{"CRIO_HOOK_FILENAME"},
+			DefaultText: defaultHookFilename,
+		},
+	}
+
+	// Update the subcommand flags with the common subcommand flags
+	setup.Flags = append([]cli.Flag{}, commonFlags...)
+	cleanup.Flags = append([]cli.Flag{}, commonFlags...)
+
+	// Run the top-level CLI
+	if err := c.Run(os.Args); err != nil {
+		log.Fatal(fmt.Errorf("error: %v", err))
+	}
+}
+
+// Setup installs the prestart hook required to launch GPU-enabled containers
+func Setup(c *cli.Context) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	err := os.MkdirAll(hooksDirFlag, 0755)
+	if err != nil {
+		return fmt.Errorf("error creating hooks directory %v: %v", hooksDirFlag, err)
+	}
+
+	hookPath := getHookPath(hooksDirFlag, hookFilenameFlag)
+	err = createHook(tooklitDirArg, hookPath)
+	if err != nil {
+		return fmt.Errorf("error creating hook: %v", err)
+	}
+
+	return nil
+}
+
+// Cleanup removes the specified prestart hook
+func Cleanup(c *cli.Context) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	hookPath := getHookPath(hooksDirFlag, hookFilenameFlag)
+	err := os.Remove(hookPath)
+	if err != nil {
+		return fmt.Errorf("error removing hook '%v': %v", hookPath, err)
+	}
+
+	return nil
+}
+
+// ParseArgs parses the command line arguments to the CLI
+func ParseArgs(c *cli.Context) error {
+	args := c.Args()
+
+	log.Infof("Parsing arguments: %v", args.Slice())
+	if c.NArg() != 1 {
+		return fmt.Errorf("incorrect number of arguments")
+	}
+	tooklitDirArg = args.Get(0)
+	log.Infof("Successfully parsed arguments")
+
+	return nil
+}
+
+func createHook(toolkitDir string, hookPath string) error {
+	hook, err := os.Create(hookPath)
+	if err != nil {
+		return fmt.Errorf("error creating hook file '%v': %v", hookPath, err)
+	}
+	defer hook.Close()
+
+	encoder := json.NewEncoder(hook)
+	err = encoder.Encode(generateOciHook(tooklitDirArg))
+	if err != nil {
+		return fmt.Errorf("error writing hook file '%v': %v", hookPath, err)
+	}
+	return nil
+}
+
+func getHookPath(hooksDir string, hookFilename string) string {
+	return filepath.Join(hooksDir, hookFilename)
+}
+
+func generateOciHook(toolkitDir string) hooks.Hook {
+	hookPath := filepath.Join(toolkitDir, "nvidia-container-toolkit")
+	envPath := "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:" + toolkitDir
+	always := true
+
+	hook := hooks.Hook{
+		Version: "1.0.0",
+		Stages:  []string{"prestart"},
+		Hook: rspec.Hook{
+			Path: hookPath,
+			Args: []string{"nvidia-container-toolkit", "prestart"},
+			Env:  []string{envPath},
+		},
+		When: hooks.When{
+			Always:   &always,
+			Commands: []string{".*"},
+		},
+	}
+	return hook
+}

tools/container/docker/docker.go

@@ -0,0 +1,462 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+)
+
+const (
+	nvidiaRuntimeName               = "nvidia"
+	nvidiaRuntimeBinary             = "nvidia-container-runtime"
+	nvidiaExperimentalRuntimeName   = "nvidia-experimental"
+	nvidiaExperimentalRuntimeBinary = "nvidia-container-runtime-experimental"
+
+	defaultConfig       = "/etc/docker/daemon.json"
+	defaultSocket       = "/var/run/docker.sock"
+	defaultSetAsDefault = true
+	// defaultRuntimeName specifies the NVIDIA runtime to be use as the default runtime if setting the default runtime is enabled
+	defaultRuntimeName = nvidiaRuntimeName
+
+	reloadBackoff     = 5 * time.Second
+	maxReloadAttempts = 6
+
+	defaultDockerRuntime  = "runc"
+	socketMessageToGetPID = "GET /info HTTP/1.0\r\n\r\n"
+)
+
+// nvidiaRuntimeBinaries defines a map of runtime names to binary names
+var nvidiaRuntimeBinaries = map[string]string{
+	nvidiaRuntimeName:             nvidiaRuntimeBinary,
+	nvidiaExperimentalRuntimeName: nvidiaExperimentalRuntimeBinary,
+}
+
+// options stores the configuration from the command line or environment variables
+type options struct {
+	config       string
+	socket       string
+	runtimeName  string
+	setAsDefault bool
+	runtimeDir   string
+}
+
+func main() {
+	options := options{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "docker"
+	c.Usage = "Update docker config with the nvidia runtime"
+	c.Version = "0.1.0"
+
+	// Create the 'setup' subcommand
+	setup := cli.Command{}
+	setup.Name = "setup"
+	setup.Usage = "Trigger docker config to be updated"
+	setup.ArgsUsage = "<runtime_dirname>"
+	setup.Action = func(c *cli.Context) error {
+		return Setup(c, &options)
+	}
+
+	// Create the 'cleanup' subcommand
+	cleanup := cli.Command{}
+	cleanup.Name = "cleanup"
+	cleanup.Usage = "Trigger any updates made to docker config to be undone"
+	cleanup.ArgsUsage = "<runtime_dirname>"
+	cleanup.Action = func(c *cli.Context) error {
+		return Cleanup(c, &options)
+	}
+
+	// Register the subcommands with the top-level CLI
+	c.Commands = []*cli.Command{
+		&setup,
+		&cleanup,
+	}
+
+	// Setup common flags across both subcommands. All subcommands get the same
+	// set of flags even if they don't use some of them. This is so that we
+	// only require the user to specify one set of flags for both 'startup'
+	// and 'cleanup' to simplify things.
+	commonFlags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config",
+			Aliases:     []string{"c"},
+			Usage:       "Path to docker config file",
+			Value:       defaultConfig,
+			Destination: &options.config,
+			EnvVars:     []string{"DOCKER_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "socket",
+			Aliases:     []string{"s"},
+			Usage:       "Path to the docker socket file",
+			Value:       defaultSocket,
+			Destination: &options.socket,
+			EnvVars:     []string{"DOCKER_SOCKET"},
+		},
+		// The flags below are only used by the 'setup' command.
+		&cli.StringFlag{
+			Name:        "runtime-name",
+			Aliases:     []string{"r"},
+			Usage:       "Specify the name of the `nvidia` runtime. If set-as-default is selected, the runtime is used as the default runtime.",
+			Value:       defaultRuntimeName,
+			Destination: &options.runtimeName,
+			EnvVars:     []string{"DOCKER_RUNTIME_NAME"},
+		},
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Aliases:     []string{"d"},
+			Usage:       "Set the `nvidia` runtime as the default runtime. If --runtime-name is specified as `nvidia-experimental` the experimental runtime is set as the default runtime instead",
+			Value:       defaultSetAsDefault,
+			Destination: &options.setAsDefault,
+			EnvVars:     []string{"DOCKER_SET_AS_DEFAULT"},
+			Hidden:      true,
+		},
+	}
+
+	// Update the subcommand flags with the common subcommand flags
+	setup.Flags = append([]cli.Flag{}, commonFlags...)
+	cleanup.Flags = append([]cli.Flag{}, commonFlags...)
+
+	// Run the top-level CLI
+	if err := c.Run(os.Args); err != nil {
+		log.Errorf("Error running docker configuration: %v", err)
+		os.Exit(1)
+	}
+}
+
+// Setup updates docker configuration to include the nvidia runtime and reloads it
+func Setup(c *cli.Context, o *options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	runtimeDir, err := ParseArgs(c)
+	if err != nil {
+		return fmt.Errorf("unable to parse args: %v", err)
+	}
+	o.runtimeDir = runtimeDir
+
+	cfg, err := LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = UpdateConfig(cfg, o)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = FlushConfig(cfg, o.config)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = SignalDocker(o.socket)
+	if err != nil {
+		return fmt.Errorf("unable to signal docker: %v", err)
+	}
+
+	log.Infof("Completed 'setup' for %v", c.App.Name)
+
+	return nil
+}
+
+// Cleanup reverts docker configuration to remove the nvidia runtime and reloads it
+func Cleanup(c *cli.Context, o *options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	_, err := ParseArgs(c)
+	if err != nil {
+		return fmt.Errorf("unable to parse args: %v", err)
+	}
+
+	cfg, err := LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = RevertConfig(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = FlushConfig(cfg, o.config)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = SignalDocker(o.socket)
+	if err != nil {
+		return fmt.Errorf("unable to signal docker: %v", err)
+	}
+
+	log.Infof("Completed 'cleanup' for %v", c.App.Name)
+
+	return nil
+}
+
+// ParseArgs parses the command line arguments to the CLI
+func ParseArgs(c *cli.Context) (string, error) {
+	args := c.Args()
+
+	log.Infof("Parsing arguments: %v", args.Slice())
+	if args.Len() != 1 {
+		return "", fmt.Errorf("incorrect number of arguments")
+	}
+	runtimeDir := args.Get(0)
+	log.Infof("Successfully parsed arguments")
+
+	return runtimeDir, nil
+}
+
+// LoadConfig loads the docker config from disk
+func LoadConfig(config string) (map[string]interface{}, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	cfg := make(map[string]interface{})
+
+	if os.IsNotExist(err) {
+		log.Infof("Config file does not exist, creating new one")
+		return cfg, nil
+	}
+
+	readBytes, err := ioutil.ReadFile(config)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read config: %v", err)
+	}
+
+	reader := bytes.NewReader(readBytes)
+	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+	return cfg, nil
+}
+
+// UpdateConfig updates the docker config to include the nvidia runtimes
+func UpdateConfig(config map[string]interface{}, o *options) error {
+	defaultRuntime := o.getDefaultRuntime()
+	if defaultRuntime != "" {
+		config["default-runtime"] = defaultRuntime
+	}
+
+	runtimes := make(map[string]interface{})
+	if _, exists := config["runtimes"]; exists {
+		runtimes = config["runtimes"].(map[string]interface{})
+	}
+
+	for name, rt := range o.runtimes() {
+		runtimes[name] = rt
+	}
+
+	config["runtimes"] = runtimes
+	return nil
+}
+
+//RevertConfig reverts the docker config to remove the nvidia runtime
+func RevertConfig(config map[string]interface{}) error {
+	if _, exists := config["default-runtime"]; exists {
+		defaultRuntime := config["default-runtime"].(string)
+		if _, exists := nvidiaRuntimeBinaries[defaultRuntime]; exists {
+			config["default-runtime"] = defaultDockerRuntime
+		}
+	}
+
+	if _, exists := config["runtimes"]; exists {
+		runtimes := config["runtimes"].(map[string]interface{})
+
+		for name := range nvidiaRuntimeBinaries {
+			delete(runtimes, name)
+		}
+
+		if len(runtimes) == 0 {
+			delete(config, "runtimes")
+		}
+	}
+	return nil
+}
+
+// FlushConfig flushes the updated/reverted config out to disk
+func FlushConfig(cfg map[string]interface{}, config string) error {
+	log.Infof("Flushing config")
+
+	output, err := json.MarshalIndent(cfg, "", "    ")
+	if err != nil {
+		return fmt.Errorf("unable to convert to JSON: %v", err)
+	}
+
+	switch len(output) {
+	case 0:
+		err := os.Remove(config)
+		if err != nil {
+			return fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		log.Infof("Config empty, removing file")
+	default:
+		f, err := os.Create(config)
+		if err != nil {
+			return fmt.Errorf("unable to open %v for writing: %v", config, err)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(string(output))
+		if err != nil {
+			return fmt.Errorf("unable to write output: %v", err)
+		}
+	}
+
+	log.Infof("Successfully flushed config")
+
+	return nil
+}
+
+// SignalDocker sends a SIGHUP signal to docker daemon
+func SignalDocker(socket string) error {
+	log.Infof("Sending SIGHUP signal to docker")
+
+	// Wrap the logic to perform the SIGHUP in a function so we can retry it on failure
+	retriable := func() error {
+		conn, err := net.Dial("unix", socket)
+		if err != nil {
+			return fmt.Errorf("unable to dial: %v", err)
+		}
+		defer conn.Close()
+
+		sconn, err := conn.(*net.UnixConn).SyscallConn()
+		if err != nil {
+			return fmt.Errorf("unable to get syscall connection: %v", err)
+		}
+
+		err1 := sconn.Control(func(fd uintptr) {
+			err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
+		})
+		if err1 != nil {
+			return fmt.Errorf("unable to issue call on socket fd: %v", err1)
+		}
+		if err != nil {
+			return fmt.Errorf("unable to SetsockoptInt on socket fd: %v", err)
+		}
+
+		_, _, err = conn.(*net.UnixConn).WriteMsgUnix([]byte(socketMessageToGetPID), nil, nil)
+		if err != nil {
+			return fmt.Errorf("unable to WriteMsgUnix on socket fd: %v", err)
+		}
+
+		oob := make([]byte, 1024)
+		_, oobn, _, _, err := conn.(*net.UnixConn).ReadMsgUnix(nil, oob)
+		if err != nil {
+			return fmt.Errorf("unable to ReadMsgUnix on socket fd: %v", err)
+		}
+
+		oob = oob[:oobn]
+		scm, err := syscall.ParseSocketControlMessage(oob)
+		if err != nil {
+			return fmt.Errorf("unable to ParseSocketControlMessage from message received on socket fd: %v", err)
+		}
+
+		ucred, err := syscall.ParseUnixCredentials(&scm[0])
+		if err != nil {
+			return fmt.Errorf("unable to ParseUnixCredentials from message received on socket fd: %v", err)
+		}
+
+		err = syscall.Kill(int(ucred.Pid), syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("unable to send SIGHUP to 'docker' process: %v", err)
+		}
+
+		return nil
+	}
+
+	// Try to send a SIGHUP up to maxReloadAttempts times
+	var err error
+	for i := 0; i < maxReloadAttempts; i++ {
+		err = retriable()
+		if err == nil {
+			break
+		}
+		if i == maxReloadAttempts-1 {
+			break
+		}
+		log.Warnf("Error signaling docker, attempt %v/%v: %v", i+1, maxReloadAttempts, err)
+		time.Sleep(reloadBackoff)
+	}
+	if err != nil {
+		log.Warnf("Max retries reached %v/%v, aborting", maxReloadAttempts, maxReloadAttempts)
+		return err
+	}
+
+	log.Infof("Successfully signaled docker")
+
+	return nil
+}
+
+// getDefaultRuntime returns the default runtime for the configured options.
+// If the configuration is invalid or the default runtimes should not be set
+// the empty string is returned.
+func (o options) getDefaultRuntime() string {
+	if o.setAsDefault == false {
+		return ""
+	}
+
+	return o.runtimeName
+}
+
+// runtimes returns the docker runtime definitions for the supported nvidia runtimes
+// for the given options. This includes the path with the options runtimeDir applied
+func (o options) runtimes() map[string]interface{} {
+	runtimes := make(map[string]interface{})
+	for r, bin := range o.getRuntimeBinaries() {
+		runtimes[r] = map[string]interface{}{
+			"path": bin,
+			"args": []string{},
+		}
+	}
+	return runtimes
+}
+
+// getRuntimeBinaries returns a map of runtime names to binary paths. This includes the
+// renaming of the `nvidia` runtime as per the --runtime-class command line flag.
+func (o options) getRuntimeBinaries() map[string]string {
+	runtimeBinaries := make(map[string]string)
+
+	for rt, bin := range nvidiaRuntimeBinaries {
+		runtime := rt
+		if o.runtimeName != "" && o.runtimeName != nvidiaExperimentalRuntimeName && runtime == defaultRuntimeName {
+			runtime = o.runtimeName
+		}
+
+		runtimeBinaries[runtime] = filepath.Join(o.runtimeDir, bin)
+	}
+
+	return runtimeBinaries
+}

tools/container/docker/docker_test.go

@@ -0,0 +1,423 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateConfigDefaultRuntime(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		setAsDefault               bool
+		runtimeName                string
+		expectedDefaultRuntimeName interface{}
+	}{
+		{},
+		{
+			setAsDefault:               false,
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "NAME",
+			expectedDefaultRuntimeName: "NAME",
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "nvidia-experimental",
+			expectedDefaultRuntimeName: "nvidia-experimental",
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "nvidia",
+			expectedDefaultRuntimeName: "nvidia",
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			setAsDefault: tc.setAsDefault,
+			runtimeName:  tc.runtimeName,
+			runtimeDir:   runtimeDir,
+		}
+
+		config := map[string]interface{}{}
+
+		err := UpdateConfig(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		defaultRuntimeName := config["default-runtime"]
+		require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
+	}
+}
+
+func TestUpdateConfig(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		config         map[string]interface{}
+		setAsDefault   bool
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			config:       map[string]interface{}{},
+			setAsDefault: false,
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config:       map[string]interface{}{},
+			setAsDefault: false,
+			runtimeName:  "NAME",
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"NAME": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config:       map[string]interface{}{},
+			setAsDefault: false,
+			runtimeName:  "nvidia-experimental",
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			setAsDefault: false,
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "runc",
+			},
+			setAsDefault: true,
+			runtimeName:  "nvidia",
+			expectedConfig: map[string]interface{}{
+				"default-runtime": "nvidia",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "runc",
+			},
+			setAsDefault: true,
+			runtimeName:  "nvidia-experimental",
+			expectedConfig: map[string]interface{}{
+				"default-runtime": "nvidia-experimental",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		options := &options{
+			setAsDefault: tc.setAsDefault,
+			runtimeName:  tc.runtimeName,
+			runtimeDir:   runtimeDir,
+		}
+		err := UpdateConfig(tc.config, options)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		configContent, err := json.MarshalIndent(tc.config, "", "    ")
+		require.NoError(t, err)
+
+		expectedContent, err := json.MarshalIndent(tc.expectedConfig, "", "    ")
+		require.NoError(t, err)
+
+		require.EqualValues(t, string(expectedContent), string(configContent), "%d: %v", i, tc)
+	}
+}
+
+func TestRevertConfig(t *testing.T) {
+	testCases := []struct {
+		config         map[string]interface{}
+		expectedConfig map[string]interface{}
+	}{
+		{
+			config:         map[string]interface{}{},
+			expectedConfig: map[string]interface{}{},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{},
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "nvidia",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"default-runtime": "runc",
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "not-nvidia",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"default-runtime": "not-nvidia",
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		err := RevertConfig(tc.config)
+
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		configContent, err := json.MarshalIndent(tc.config, "", "    ")
+		require.NoError(t, err)
+
+		expectedContent, err := json.MarshalIndent(tc.expectedConfig, "", "    ")
+		require.NoError(t, err)
+
+		require.EqualValues(t, string(expectedContent), string(configContent), "%d: %v", i, tc)
+	}
+}
+
+func TestFlagsDefaultRuntime(t *testing.T) {
+	testCases := []struct {
+		setAsDefault bool
+		runtimeName  string
+		expected     string
+	}{
+		{
+			expected: "",
+		},
+		{
+			runtimeName: "not-bool",
+			expected:    "",
+		},
+		{
+			setAsDefault: false,
+			runtimeName:  "nvidia",
+			expected:     "",
+		},
+		{
+			setAsDefault: true,
+			runtimeName:  "nvidia",
+			expected:     "nvidia",
+		},
+		{
+			setAsDefault: true,
+			runtimeName:  "nvidia-experimental",
+			expected:     "nvidia-experimental",
+		},
+	}
+
+	for i, tc := range testCases {
+		f := options{
+			setAsDefault: tc.setAsDefault,
+			runtimeName:  tc.runtimeName,
+		}
+
+		require.Equal(t, tc.expected, f.getDefaultRuntime(), "%d: %v", i, tc)
+	}
+}

tools/container/nvidia-toolkit/run.go

@@ -0,0 +1,290 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+	unix "golang.org/x/sys/unix"
+)
+
+const (
+	runDir         = "/run/nvidia"
+	pidFile        = runDir + "/toolkit.pid"
+	toolkitCommand = "toolkit"
+	toolkitSubDir  = "toolkit"
+
+	defaultToolkitArgs = ""
+	defaultRuntime     = "docker"
+	defaultRuntimeArgs = ""
+)
+
+var availableRuntimes = map[string]struct{}{"docker": {}, "crio": {}, "containerd": {}}
+
+var waitingForSignal = make(chan bool, 1)
+var signalReceived = make(chan bool, 1)
+
+var destinationArg string
+var noDaemonFlag bool
+var toolkitArgsFlag string
+var runtimeFlag string
+var runtimeArgsFlag string
+
+// Version defines the CLI version. This is set at build time using LD FLAGS
+var Version = "development"
+
+func main() {
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "nvidia-toolkit"
+	c.Usage = "Install the nvidia-container-toolkit for use by a given runtime"
+	c.UsageText = "DESTINATION [-n | --no-daemon] [-t | --toolkit-args] [-r | --runtime] [-u | --runtime-args]"
+	c.Description = "DESTINATION points to the host path underneath which the nvidia-container-toolkit should be installed.\nIt will be installed at ${DESTINATION}/toolkit"
+	c.Version = Version
+	c.Action = Run
+
+	// Setup flags for the CLI
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "no-daemon",
+			Aliases:     []string{"n"},
+			Usage:       "terminate immediatly after setting up the runtime. Note that no cleanup will be performed",
+			Destination: &noDaemonFlag,
+			EnvVars:     []string{"NO_DAEMON"},
+		},
+		&cli.StringFlag{
+			Name:        "toolkit-args",
+			Aliases:     []string{"t"},
+			Usage:       "arguments to pass to the underlying 'toolkit' command",
+			Value:       defaultToolkitArgs,
+			Destination: &toolkitArgsFlag,
+			EnvVars:     []string{"TOOLKIT_ARGS"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Aliases:     []string{"r"},
+			Usage:       "the runtime to setup on this node. One of {'docker', 'crio', 'containerd'}",
+			Value:       defaultRuntime,
+			Destination: &runtimeFlag,
+			EnvVars:     []string{"RUNTIME"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-args",
+			Aliases:     []string{"u"},
+			Usage:       "arguments to pass to 'docker', 'crio', or 'containerd' setup command",
+			Value:       defaultRuntimeArgs,
+			Destination: &runtimeArgsFlag,
+			EnvVars:     []string{"RUNTIME_ARGS"},
+		},
+	}
+
+	// Run the CLI
+	log.Infof("Starting %v", c.Name)
+
+	remainingArgs, err := ParseArgs(os.Args)
+	if err != nil {
+		log.Errorf("Error: unable to parse arguments: %v", err)
+		os.Exit(1)
+	}
+
+	if err := c.Run(remainingArgs); err != nil {
+		log.Errorf("error running nvidia-toolkit: %v", err)
+		os.Exit(1)
+	}
+
+	log.Infof("Completed %v", c.Name)
+}
+
+// Run runs the core logic of the CLI
+func Run(c *cli.Context) error {
+	err := verifyFlags()
+	if err != nil {
+		return fmt.Errorf("unable to verify flags: %v", err)
+	}
+
+	err = initialize()
+	if err != nil {
+		return fmt.Errorf("unable to initialize: %v", err)
+	}
+	defer shutdown()
+
+	err = installToolkit()
+	if err != nil {
+		return fmt.Errorf("unable to install toolkit: %v", err)
+	}
+
+	err = setupRuntime()
+	if err != nil {
+		return fmt.Errorf("unable to setup runtime: %v", err)
+	}
+
+	if !noDaemonFlag {
+		err = waitForSignal()
+		if err != nil {
+			return fmt.Errorf("unable to wait for signal: %v", err)
+		}
+
+		err = cleanupRuntime()
+		if err != nil {
+			return fmt.Errorf("unable to cleanup runtime: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// ParseArgs parses the command line arguments and returns the remaining arguments
+func ParseArgs(args []string) ([]string, error) {
+	log.Infof("Parsing arguments")
+
+	numPositionalArgs := 2 // Includes command itself
+
+	if len(args) < numPositionalArgs {
+		return nil, fmt.Errorf("missing arguments")
+	}
+
+	for _, arg := range args {
+		if arg == "--help" || arg == "-h" {
+			return []string{args[0], arg}, nil
+		}
+		if arg == "--version" || arg == "-v" {
+			return []string{args[0], arg}, nil
+		}
+	}
+
+	for _, arg := range args[:numPositionalArgs] {
+		if strings.HasPrefix(arg, "-") {
+			return nil, fmt.Errorf("unexpected flag where argument should be")
+		}
+	}
+
+	for _, arg := range args[numPositionalArgs:] {
+		if !strings.HasPrefix(arg, "-") {
+			return nil, fmt.Errorf("unexpected argument where flag should be")
+		}
+	}
+
+	destinationArg = args[1]
+
+	return append([]string{args[0]}, args[numPositionalArgs:]...), nil
+}
+
+func verifyFlags() error {
+	log.Infof("Verifying Flags")
+	if _, exists := availableRuntimes[runtimeFlag]; !exists {
+		return fmt.Errorf("unknown runtime: %v", runtimeFlag)
+	}
+	return nil
+}
+
+func initialize() error {
+	log.Infof("Initializing")
+
+	f, err := os.Create(pidFile)
+	if err != nil {
+		return fmt.Errorf("unable to create pidfile: %v", err)
+	}
+
+	err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
+	if err != nil {
+		log.Warnf("Unable to get exclusive lock on '%v'", pidFile)
+		log.Warnf("This normally means an instance of the NVIDIA toolkit Container is already running, aborting")
+		return fmt.Errorf("unable to get flock on pidfile: %v", err)
+	}
+
+	_, err = f.WriteString(fmt.Sprintf("%v\n", os.Getpid()))
+	if err != nil {
+		return fmt.Errorf("unable to write PID to pidfile: %v", err)
+	}
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGPIPE, syscall.SIGTERM)
+	go func() {
+		<-sigs
+		select {
+		case <-waitingForSignal:
+			signalReceived <- true
+		default:
+			log.Infof("Signal received, exiting early")
+			shutdown()
+			os.Exit(0)
+		}
+	}()
+
+	return nil
+}
+
+func installToolkit() error {
+	toolkitDir := filepath.Join(destinationArg, toolkitSubDir)
+
+	log.Infof("Installing toolkit")
+
+	cmdline := fmt.Sprintf("%v install %v %v\n", toolkitCommand, toolkitArgsFlag, toolkitDir)
+	cmd := exec.Command("sh", "-c", cmdline)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error running %v command: %v", toolkitCommand, err)
+	}
+
+	return nil
+}
+
+func setupRuntime() error {
+	toolkitDir := filepath.Join(destinationArg, toolkitSubDir)
+
+	log.Infof("Setting up runtime")
+
+	cmdline := fmt.Sprintf("%v setup %v %v\n", runtimeFlag, runtimeArgsFlag, toolkitDir)
+
+	cmd := exec.Command("sh", "-c", cmdline)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error running %v command: %v", runtimeFlag, err)
+	}
+
+	return nil
+}
+
+func waitForSignal() error {
+	log.Infof("Waiting for signal")
+	waitingForSignal <- true
+	<-signalReceived
+	return nil
+}
+
+func cleanupRuntime() error {
+	toolkitDir := filepath.Join(destinationArg, toolkitSubDir)
+
+	log.Infof("Cleaning up Runtime")
+
+	cmdline := fmt.Sprintf("%v cleanup %v %v\n", runtimeFlag, runtimeArgsFlag, toolkitDir)
+
+	cmd := exec.Command("sh", "-c", cmdline)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error running %v command: %v", runtimeFlag, err)
+	}
+
+	return nil
+}
+
+func shutdown() {
+	log.Infof("Shutting Down")
+
+	err := os.Remove(pidFile)
+	if err != nil {
+		log.Warnf("Unable to remove pidfile: %v", err)
+	}
+}