Merge branch 'match-toolkit-tag' into 'master'

Ensure that package tags for components match

See merge request nvidia/container-toolkit/container-toolkit!76

.common-ci.yml

@@ -18,8 +18,7 @@ default:
       command: ["--experimental"]
 
 variables:
-  IMAGE: "${CI_REGISTRY_IMAGE}"
-  IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
+  GIT_SUBMODULE_STRATEGY: recursive
   BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}"
 
 stages:
@@ -28,10 +27,12 @@ stages:
   - go-checks
   - go-build
   - unit-tests
-  - build
-  - build-long
+  - package-build
+  - image-build
+  - test
   - scan
   - release
+  - build-all
 
 build-dev-image:
   stage: image
@@ -95,3 +96,314 @@ unit-tests:
   script:
     - make coverage
 
+
+# Define the distribution targets
+.dist-centos7:
+  variables:
+    DIST: centos7
+
+.dist-centos8:
+  variables:
+    DIST: centos8
+
+.dist-ubi8:
+  variables:
+    DIST: ubi8
+
+.dist-ubuntu18.04:
+  variables:
+    DIST: ubuntu18.04
+
+.arch-aarch64:
+  variables:
+    ARCH: aarch64
+
+.arch-amd64:
+  variables:
+    ARCH: amd64
+
+.arch-arm64:
+  variables:
+    ARCH: arm64
+
+.arch-ppc64le:
+  variables:
+    ARCH: ppc64le
+
+.arch-x86_64:
+  variables:
+    ARCH: x86_64
+
+# Define the package build helpers
+.multi-arch-build:
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
+
+.package-artifacts:
+  variables:
+    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
+    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
+    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
+
+.package-build:
+  extends:
+    - .multi-arch-build
+    - .package-artifacts
+  stage: package-build
+  script:
+    - ./scripts/release.sh ${DIST}-${ARCH}
+
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
+
+# Define the package build targets
+package-ubuntu18.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-amd64
+
+package-ubuntu18.04-arm64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-arm64
+
+package-ubuntu18.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-ppc64le
+
+package-centos7-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-x86_64
+
+package-centos8-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-x86_64
+
+# Define the image build targets
+.image-build:
+  stage: image-build
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache bash make
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+
+image-centos7:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos7
+  needs:
+    - package-centos7-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-centos8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos8
+  needs:
+    - package-centos8-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-ubi8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubi8
+  needs:
+    # Note: The ubi8 image currently uses the centos7 packages
+    - package-centos7-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-ubuntu18.04:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubuntu18.04
+  needs:
+    - package-ubuntu18.04-amd64
+    # TODO: These will be required once we generate multi-arch images
+    # - package-ubuntu18.04-arm64
+    # - package-ubuntu18.04-ppc64le
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+# Define test helpers
+.integration:
+  stage: test
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache make bash jq
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
+  script:
+    - make -f build/container/Makefile test-${DIST}
+
+.test:toolkit:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "toolkit"
+
+.test:docker:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "docker"
+
+.test:containerd:
+  # TODO: The containerd tests fail due to issues with SIGHUP.
+  # Until this is resolved with retry up to twice and allow failure here.
+  retry: 2
+  allow_failure: true
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "containerd"
+
+.test:crio:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "crio"
+
+# Define the test targets
+test-toolkit-ubuntu18.04:
+  extends:
+    - .test:toolkit
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-containerd-ubuntu18.04:
+  extends:
+    - .test:containerd
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-crio-ubuntu18.04:
+  extends:
+    - .test:crio
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-docker-ubuntu18.04:
+  extends:
+    - .test:docker
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+# .release forms the base of the deployment jobs which push images to the CI registry.
+# This is extended with the version to be deployed (e.g. the SHA or TAG) and the
+# target os.
+.release:
+  stage:
+    release
+  variables:
+    # Define the source image for the release
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    # OUT_IMAGE_VERSION is overridden for external releases
+    OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
+  stage: release
+  before_script:
+    # We ensure that the OUT_IMAGE_VERSION is set
+    - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
+
+    # In the case where we are deploying a different version to the CI_COMMIT_SHA, we
+    # need to tag the image.
+    # Note: a leading 'v' is stripped from the version if present
+    - apk add --no-cache make bash
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
+  script:
+    - docker tag "${IMAGE_NAME}:${VERSION}-${DIST}" "${OUT_IMAGE_NAME}:${OUT_IMAGE_VERSION}-${DIST}"
+    # Log in to the "output" registry, tag the image and push the image
+    - 'echo "Logging in to output registry ${OUT_REGISTRY}"'
+    - docker logout
+    - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}"
+    - make IMAGE_NAME=${OUT_IMAGE_NAME} VERSION=${OUT_IMAGE_VERSION} -f build/container/Makefile push-${DIST}
+
+# Define a staging release step that pushes an image to an internal "staging" repository
+# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
+# outside of the release process.
+.release:staging:
+  extends:
+    - .release
+  variables:
+    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    OUT_REGISTRY: "${CI_REGISTRY}"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit"
+
+# Define an external release step that pushes an image to an external repository.
+# This includes a devlopment image off master.
+.release:external:
+  extends:
+    - .release
+  rules:
+    - if: $CI_COMMIT_TAG
+      variables:
+        OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}"
+    - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH
+      variables:
+        OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
+
+# Define the release jobs
+release:staging-centos7:
+  extends:
+    - .release:staging
+    - .dist-centos7
+  needs:
+    - image-centos7
+
+release:staging-centos8:
+  extends:
+    - .release:staging
+    - .dist-centos8
+  needs:
+    - image-centos8
+
+release:staging-ubi8:
+  extends:
+    - .release:staging
+    - .dist-ubi8
+  needs:
+    - image-ubi8
+
+release:staging-ubuntu18.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu18.04
+  needs:
+    - test-toolkit-ubuntu18.04
+    - test-containerd-ubuntu18.04
+    - test-crio-ubuntu18.04
+    - test-docker-ubuntu18.04

.dockerignore

@@ -1,2 +1,2 @@
 .git
-dist
+/shared-*

.gitignore

@@ -5,3 +5,4 @@ dist
 /test/output/
 /nvidia-container-runtime
 /nvidia-container-toolkit
+/shared-*

.gitlab-ci.yml

@@ -15,107 +15,46 @@
 include:
   - .common-ci.yml
 
-.build-setup:
-  before_script:
-  - apk update
-  - apk upgrade
-  - apk add coreutils build-base sed git bash make
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes
-
-# build-one jobs build packages for a single OS / ARCH combination.
-#
-# They are run during the first stage of the pipeline as a smoke test to ensure
-# that we can successfully build packages on all of our architectures for a
-# single OS. They are triggered on any change to an MR. No artifacts are
-# produced as part of build-one jobs.
-.build-one-setup:
-  extends:
-    - .build-setup
-  stage: build
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-
 # build-all jobs build packages for every OS / ARCH combination we support.
 #
 # They are run under two conditions:
 # 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0)
 # 2) Manually by a reviewer just before merging a MR.
-#
-# Unlike build-one jobs, it takes a long time to build the full suite
-# OS / ARCH combinations, so this is optimized to only run once per MR
-# (assuming it all passes). A full set of artifacts including the packages
-# built for each OS / ARCH are produced as a result of these jobs.
-.build-all-setup:
+.build-all-for-arch:
+  variables:
+    # Setting DIST=docker invokes the docker- release targets
+    DIST: docker
   extends:
-    - .build-setup
-  stage: build-long
+    - .package-build
+  stage: build-all
   timeout: 2h 30m
   rules:
     - if: $CI_COMMIT_TAG
       when: always
-    - if: $CI_MERGE_REQUEST_ID
-      when: always
-
-  variables:
-    ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}"
-    ARTIFACTS_DIR: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}"
-    DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_DIR}"
-
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_DIR}
-
-# The full set of build-one jobs organizes to build
-# ubuntu18.04 in parallel on each of our supported ARCHs.
-build-one-amd64:
-  extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-amd64
-  rules:
-    - when: always
-
-build-one-ppc64le:
-  extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-ppc64le
-
-build-one-arm64:
-  extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-arm64
 
 # The full set of build-all jobs organized to
 # have builds for each ARCH run in parallel.
 build-all-amd64:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-amd64
+    - .build-all-for-arch
+    - .arch-amd64
 
 build-all-x86_64:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-x86_64
+    - .build-all-for-arch
+    - .arch-x86_64
 
 build-all-ppc64le:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-ppc64le
+    - .build-all-for-arch
+    - .arch-ppc64le
 
 build-all-arm64:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-arm64
+    - .build-all-for-arch
+    - .arch-arm64
 
 build-all-aarch64:
   extends:
-    - .build-all-setup
-  script:
-    - make docker-aarch64
+    - .build-all-for-arch
+    - .arch-aarch64

.nvidia-ci.yml

@@ -0,0 +1,174 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include:
+  - local: '.common-ci.yml'
+
+default:
+  tags:
+    - cnt
+    - container-dev
+    - docker/multi-arch
+    - docker/privileged
+    - os/linux
+    - type/docker
+
+variables:
+  DOCKER_DRIVER: overlay2
+  DOCKER_TLS_CERTDIR: "/certs"
+  # Release "devel"-tagged images off the master branch
+  RELEASE_DEVEL_BRANCH: "master"
+  DEVEL_RELEASE_IMAGE_VERSION: "devel"
+  # On the multi-arch builder we don't need the qemu setup.
+  SKIP_QEMU_SETUP: "1"
+
+# We skip the integration tests for the internal CI:
+.integration:
+  stage: test
+  before_script:
+    - echo "Skipped in internal CI"
+  script:
+    - echo "Skipped in internal CI"
+
+# The .scan step forms the base of the image scan operation performed before releasing
+# images.
+.scan:
+  stage: scan
+  image: "${PULSE_IMAGE}"
+  variables:
+    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
+    IMAGE_ARCHIVE: "container-toolkit.tar"
+  rules:
+    - if: $CI_COMMIT_MESSAGE =~ /\[skip[ _-]scans?\]/i
+      when: never
+    - if: $SKIP_SCANS
+      when: never
+    - if: $CI_COMMIT_TAG == null && $CI_COMMIT_BRANCH != $RELEASE_DEVEL_BRANCH
+      allow_failure: true
+  before_script:
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    # TODO: We should specify the architecture here and scan all architectures
+    - docker pull "${IMAGE}"
+    - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
+    - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
+    - >
+      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
+    - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+  script:
+    - pulse-cli -n $NSPECT_ID --pss $PSS_URL --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+  artifacts:
+    when: always
+    expire_in: 1 week
+    paths:
+      - pulse-cli.log
+      - licenses.json
+      - sbom.json
+      - vulns.json
+      - policy_evaluation.json
+
+# Define the scan targets
+scan-centos7:
+  extends:
+    - .scan
+    - .dist-centos7
+  needs:
+    - image-centos7
+
+scan-centos8:
+  extends:
+    - .scan
+    - .dist-centos8
+  needs:
+    - image-centos8
+
+scan-ubuntu18.04:
+  extends:
+    - .scan
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+scan-ubi8:
+  extends:
+    - .scan
+    - .dist-ubi8
+  needs:
+    - image-ubi8
+
+# Define external release helpers
+.release:ngc:
+  extends:
+    - .release:external
+  variables:
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
+    OUT_REGISTRY: "${NGC_REGISTRY}"
+    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    # TODO: For now we disable external releases
+    DOCKER: echo
+
+.release:dockerhub:
+  extends:
+    - .release:external
+  variables:
+    OUT_REGISTRY_USER: "${REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${REGISTRY_TOKEN}"
+    OUT_REGISTRY: "${DOCKERHUB_REGISTRY}"
+    OUT_IMAGE_NAME: "${REGISTRY_IMAGE}"
+
+    # TODO: For now we disable external releases
+    DOCKER: echo
+
+# Define the external release targets
+# Release to NGC
+release:ngc-centos7:
+  extends:
+    - .release:ngc
+    - .dist-centos7
+
+release:ngc-centos8:
+  extends:
+    - .release:ngc
+    - .dist-centos8
+
+release:ngc-ubuntu18:
+  extends:
+    - .release:ngc
+    - .dist-ubuntu18.04
+
+release:ngc-ubi8:
+  extends:
+    - .release:ngc
+    - .dist-ubi8
+
+# Release to Dockerhub
+release:dockerhub-centos7:
+  extends:
+    - .release:dockerhub
+    - .dist-centos7
+
+release:dockerhub-centos8:
+  extends:
+    - .release:dockerhub
+    - .dist-centos8
+
+release:dockerhub-ubuntu18:
+  extends:
+    - .release:dockerhub
+    - .dist-ubuntu18.04
+
+release:dockerhub-ubi8:
+  extends:
+    - .release:dockerhub
+    - .dist-ubi8

Makefile

@@ -18,7 +18,7 @@ DIST_DIR ?= $(CURDIR)/dist
 
 LIB_NAME := nvidia-container-toolkit
 LIB_VERSION := 1.6.0
-LIB_TAG ?= rc.1
+LIB_TAG ?= rc.3
 
 GOLANG_VERSION := 1.16.3
 MODULE := github.com/NVIDIA/nvidia-container-toolkit

build/container/Dockerfile.centos

@@ -0,0 +1,76 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+# NOTE: In cases where the libc version is a concern, we would have to use an
+# image based on the target OS to build the golang executables here -- especially
+# if cgo code is included.
+FROM golang:${GOLANG_VERSION} as build
+
+# We override the GOPATH to ensure that the binaries are installed to
+# /artifacts/bin
+ARG GOPATH=/artifacts
+
+# Install the experiemental nvidia-container-runtime
+# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
+ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+
+WORKDIR /build
+COPY . .
+
+# NOTE: Until the config utilities are properly integrated into the
+# nvidia-container-toolkit repository, these are built from the `tools` folder
+# and not `cmd`.
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+WORKDIR /artifacts/packages
+
+ARG ARTIFACTS_DIR
+COPY ${ARTIFACTS_DIR}/* /artifacts/packages/
+
+ARG PACKAGE_VERSION
+RUN yum localinstall -y \
+    libnvidia-container1-${PACKAGE_VERSION}*.rpm \
+    libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
+    nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
+
+WORKDIR /work
+
+COPY --from=build /artifacts/bin /work
+
+ENV PATH=/work:$PATH
+
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+COPY ./LICENSE /licenses/LICENSE
+
+ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Dockerfile.ubuntu

@@ -0,0 +1,82 @@
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+# NOTE: In cases where the libc version is a concern, we would have to use an
+# image based on the target OS to build the golang executables here -- especially
+# if cgo code is included.
+FROM golang:${GOLANG_VERSION} as build
+
+# We override the GOPATH to ensure that the binaries are installed to
+# /artifacts/bin
+ARG GOPATH=/artifacts
+
+# Install the experiemental nvidia-container-runtime
+# NOTE: This will be integrated into the nvidia-container-toolkit package / repo
+ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+
+WORKDIR /build
+COPY . .
+
+# NOTE: Until the config utilities are properly integrated into the
+# nvidia-container-toolkit repository, these are built from the `tools` folder
+# and not `cmd`.
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ARG DEBIAN_FRONTEND=noninteractive
+RUN apt-get update && apt-get install -y --no-install-recommends \
+    libcap2 \
+        && \
+    rm -rf /var/lib/apt/lists/*
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=all
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+WORKDIR /artifacts/packages
+
+ARG ARTIFACTS_DIR
+COPY ${ARTIFACTS_DIR}/* /artifacts/packages/
+
+ARG PACKAGE_VERSION
+RUN dpkg -i \
+    libnvidia-container1_${PACKAGE_VERSION}*.deb \
+    libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
+    nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
+
+WORKDIR /work
+
+COPY --from=build /artifacts/bin /work/
+
+ENV PATH=/work:$PATH
+
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+COPY ./LICENSE /licenses/LICENSE
+
+ENTRYPOINT ["/work/nvidia-toolkit"]

build/container/Makefile

@@ -0,0 +1,110 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+DOCKER   ?= docker
+MKDIR    ?= mkdir
+DIST_DIR ?= $(CURDIR)/dist
+
+##### Global variables #####
+
+# TODO: These should be defined ONCE and currently duplicate the version in the
+# toolkit makefile.
+LIB_VERSION := 1.6.0
+LIB_TAG := rc.3
+
+VERSION ?= $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
+
+CUDA_VERSION ?= 11.4.2
+GOLANG_VERSION ?= 1.16.4
+ifeq ($(IMAGE_NAME),)
+REGISTRY ?= nvidia
+IMAGE_NAME := $(REGISTRY)/container-toolkit
+endif
+
+IMAGE_TAG ?= $(VERSION)-$(DIST)
+IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
+
+##### Public rules #####
+DEFAULT_PUSH_TARGET := ubuntu18.04
+TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
+
+BUILD_TARGETS := $(patsubst %, build-%, $(TARGETS))
+PUSH_TARGETS := $(patsubst %, push-%, $(TARGETS))
+TEST_TARGETS := $(patsubst %, test-%, $(TARGETS))
+
+.PHONY: $(TARGETS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
+
+$(PUSH_TARGETS): push-%:
+	$(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)"
+
+# For the default push target we also push a short tag equal to the version.
+# We skip this for the development release
+DEVEL_RELEASE_IMAGE_VERSION ?= devel
+ifneq ($(strip $(VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
+push-$(DEFAULT_PUSH_TARGET): push-short
+endif
+push-short:
+	$(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(IMAGE_NAME):$(VERSION)"
+	$(DOCKER) push "$(IMAGE_NAME):$(VERSION)"
+
+
+build-%: DIST = $(*)
+build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
+
+# Use a generic build target to build the relevant images
+$(BUILD_TARGETS): build-%: $(ARTIFACTS_DIR)
+	$(DOCKER) build --pull \
+		--tag $(IMAGE) \
+		--build-arg ARTIFACTS_DIR="$(ARTIFACTS_DIR)" \
+		--build-arg BASE_DIST="$(BASE_DIST)" \
+		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
+		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
+		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
+		--build-arg VERSION="$(VERSION)" \
+		-f $(DOCKERFILE) \
+		$(CURDIR)
+
+
+ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
+
+build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
+build-ubuntu%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/amd64
+build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
+
+build-ubuntu18.04: BASE_DIST := ubuntu18.04
+build-ubuntu20.04: BASE_DIST := ubuntu20.04
+
+build-ubi8: DOCKERFILE_SUFFIX := centos
+# TODO: Update this to use the centos8 packages
+build-ubi8: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/centos7/x86_64
+build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-ubi8: BASE_DIST := ubi8
+
+build-centos%: DOCKERFILE_SUFFIX := centos
+build-centos%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/x86_64
+build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+
+build-centos7: BASE_DIST := centos7
+build-centos8: BASE_DIST := centos8
+
+# Test targets
+test-%: DIST = $(*)
+
+TEST_CASES ?= toolkit docker crio containerd
+$(TEST_TARGETS): test-%:
+	TEST_CASES="$(TEST_CASES)" bash -x $(CURDIR)/test/container/main.sh run \
+		$(CURDIR)/shared-$(*) \
+		$(IMAGE) \
+			--no-cleanup-on-error
+

build/container/README.md

@@ -0,0 +1,4 @@
+# NVIDIA Container Toolkit Container
+
+This folder contains make and docker files for building the NVIDIA Container Toolkit Container.
+

cmd/nvidia-container-runtime/nvcr.go

@@ -66,22 +66,9 @@ func (r nvidiaContainerRuntime) Exec(args []string) error {
 // modificationRequired checks the intput arguments to determine whether a modification
 // to the OCI spec is required.
 func (r nvidiaContainerRuntime) modificationRequired(args []string) bool {
-	var previousWasBundle bool
-	for _, a := range args {
-		// We check for '--bundle create' explicitly to ensure that we
-		// don't inadvertently trigger a modification if the bundle directory
-		// is specified as `create`
-		if !previousWasBundle && isBundleFlag(a) {
-			previousWasBundle = true
-			continue
-		}
-
-		if !previousWasBundle && a == "create" {
-			r.logger.Infof("'create' command detected; modification required")
-			return true
-		}
-
-		previousWasBundle = false
+	if oci.HasCreateSubcommand(args) {
+		r.logger.Infof("'create' command detected; modification required")
+		return true
 	}
 
 	r.logger.Infof("No modification required")

cmd/nvidia-container-runtime/nvcr_test.go

@@ -18,7 +18,6 @@ package main
 
 import (
 	"fmt"
-	"os"
 	"strings"
 	"testing"
 
@@ -28,35 +27,6 @@ import (
 	"github.com/stretchr/testify/require"
 )
 
-func TestArgsGetConfigFilePath(t *testing.T) {
-	wd, err := os.Getwd()
-	require.NoError(t, err)
-
-	testCases := []struct {
-		bundleDir   string
-		ociSpecPath string
-	}{
-		{
-			ociSpecPath: fmt.Sprintf("%v/config.json", wd),
-		},
-		{
-			bundleDir:   "/foo/bar",
-			ociSpecPath: "/foo/bar/config.json",
-		},
-		{
-			bundleDir:   "/foo/bar/",
-			ociSpecPath: "/foo/bar/config.json",
-		},
-	}
-
-	for i, tc := range testCases {
-		cp, err := getOCISpecFilePath(tc.bundleDir)
-
-		require.NoErrorf(t, err, "%d: %v", i, tc)
-		require.Equalf(t, tc.ociSpecPath, cp, "%d: %v", i, tc)
-	}
-}
-
 func TestAddNvidiaHook(t *testing.T) {
 	logger, logHook := testlog.NewNullLogger()
 	shim := nvidiaContainerRuntime{
@@ -185,9 +155,14 @@ func TestNvidiaContainerRuntime(t *testing.T) {
 		tc.shim.logger = logger
 		hook.Reset()
 
-		spec := &specs.Spec{}
-		ociMock := oci.NewMockSpec(spec, tc.writeError, tc.modifyError)
-
+		ociMock := &oci.SpecMock{
+			ModifyFunc: func(specModifier oci.SpecModifier) error {
+				return tc.modifyError
+			},
+			FlushFunc: func() error {
+				return tc.writeError
+			},
+		}
 		require.Equal(t, tc.shouldModify, tc.shim.modificationRequired(tc.args), "%d: %v", i, tc)
 
 		tc.shim.ociSpec = ociMock
@@ -201,18 +176,16 @@ func TestNvidiaContainerRuntime(t *testing.T) {
 		}
 
 		if tc.shouldModify {
-			require.Equal(t, 1, ociMock.MockModify.Callcount, "%d: %v", i, tc)
-			require.Equal(t, 1, nvidiaHookCount(spec.Hooks), "%d: %v", i, tc)
+			require.Equal(t, 1, len(ociMock.ModifyCalls()), "%d: %v", i, tc)
 		} else {
-			require.Equal(t, 0, ociMock.MockModify.Callcount, "%d: %v", i, tc)
-			require.Nil(t, spec.Hooks, "%d: %v", i, tc)
+			require.Equal(t, 0, len(ociMock.ModifyCalls()), "%d: %v", i, tc)
 		}
 
 		writeExpected := tc.shouldModify && tc.modifyError == nil
 		if writeExpected {
-			require.Equal(t, 1, ociMock.MockFlush.Callcount, "%d: %v", i, tc)
+			require.Equal(t, 1, len(ociMock.FlushCalls()), "%d: %v", i, tc)
 		} else {
-			require.Equal(t, 0, ociMock.MockFlush.Callcount, "%d: %v", i, tc)
+			require.Equal(t, 0, len(ociMock.FlushCalls()), "%d: %v", i, tc)
 		}
 	}
 }

cmd/nvidia-container-runtime/runtime_factory.go

@@ -18,10 +18,6 @@ package main
 
 import (
 	"fmt"
-	"os"
-	"os/exec"
-	"path/filepath"
-	"strings"
 
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
 )
@@ -54,15 +50,15 @@ func newRuntime(argv []string) (oci.Runtime, error) {
 
 // newOCISpec constructs an OCI spec for the provided arguments
 func newOCISpec(argv []string) (oci.Spec, error) {
-	bundlePath, err := getBundlePath(argv)
+	bundleDir, err := oci.GetBundleDir(argv)
 	if err != nil {
 		return nil, fmt.Errorf("error parsing command line arguments: %v", err)
 	}
+	logger.Infof("Using bundle directory: %v", bundleDir)
+
+	ociSpecPath := oci.GetSpecFilePath(bundleDir)
+	logger.Infof("Using OCI specification file path: %v", ociSpecPath)
 
-	ociSpecPath, err := getOCISpecFilePath(bundlePath)
-	if err != nil {
-		return nil, fmt.Errorf("error getting OCI specification file path: %v", err)
-	}
 	ociSpec := oci.NewSpecFromFile(ociSpecPath)
 
 	return ociSpec, nil
@@ -70,107 +66,9 @@ func newOCISpec(argv []string) (oci.Spec, error) {
 
 // newRuncRuntime locates the runc binary and wraps it in a SyscallExecRuntime
 func newRuncRuntime() (oci.Runtime, error) {
-	runtimePath, err := findRunc()
-	if err != nil {
-		return nil, fmt.Errorf("error locating runtime: %v", err)
-	}
-
-	runc, err := oci.NewSyscallExecRuntimeWithLogger(logger.Logger, runtimePath)
-	if err != nil {
-		return nil, fmt.Errorf("error constructing runtime: %v", err)
-	}
-
-	return runc, nil
-}
-
-// getBundlePath checks the specified slice of strings (argv) for a 'bundle' flag as allowed by runc.
-// The following are supported:
-// --bundle{{SEP}}BUNDLE_PATH
-// -bundle{{SEP}}BUNDLE_PATH
-// -b{{SEP}}BUNDLE_PATH
-// where {{SEP}} is either ' ' or '='
-func getBundlePath(argv []string) (string, error) {
-	var bundlePath string
-
-	for i := 0; i < len(argv); i++ {
-		param := argv[i]
-
-		parts := strings.SplitN(param, "=", 2)
-		if !isBundleFlag(parts[0]) {
-			continue
-		}
-
-		// The flag has the format --bundle=/path
-		if len(parts) == 2 {
-			bundlePath = parts[1]
-			continue
-		}
-
-		// The flag has the format --bundle /path
-		if i+1 < len(argv) {
-			bundlePath = argv[i+1]
-			i++
-			continue
-		}
-
-		// --bundle / -b was the last element of argv
-		return "", fmt.Errorf("bundle option requires an argument")
-	}
-
-	return bundlePath, nil
-}
-
-// findRunc locates runc in the path, returning the full path to the
-// binary or an error.
-func findRunc() (string, error) {
-	runtimeCandidates := []string{
+	return oci.NewLowLevelRuntimeWithLogger(
+		logger.Logger,
 		dockerRuncExecutableName,
 		runcExecutableName,
-	}
-
-	return findRuntime(runtimeCandidates)
-}
-
-func findRuntime(runtimeCandidates []string) (string, error) {
-	for _, candidate := range runtimeCandidates {
-		logger.Infof("Looking for runtime binary '%v'", candidate)
-		runcPath, err := exec.LookPath(candidate)
-		if err == nil {
-			logger.Infof("Found runtime binary '%v'", runcPath)
-			return runcPath, nil
-		}
-		logger.Warnf("Runtime binary '%v' not found: %v", candidate, err)
-	}
-
-	return "", fmt.Errorf("no runtime binary found from candidate list: %v", runtimeCandidates)
-}
-
-func isBundleFlag(arg string) bool {
-	if !strings.HasPrefix(arg, "-") {
-		return false
-	}
-
-	trimmed := strings.TrimLeft(arg, "-")
-	return trimmed == "b" || trimmed == "bundle"
-}
-
-// getOCISpecFilePath returns the expected path to the OCI specification file for the given
-// bundle directory or the current working directory if not specified.
-func getOCISpecFilePath(bundleDir string) (string, error) {
-	if bundleDir == "" {
-		logger.Infof("Bundle directory path is empty, using working directory.")
-		workingDirectory, err := os.Getwd()
-		if err != nil {
-			return "", fmt.Errorf("error getting working directory: %v", err)
-		}
-		bundleDir = workingDirectory
-	}
-
-	logger.Infof("Using bundle directory: %v", bundleDir)
-
-	OCISpecFilePath := filepath.Join(bundleDir, ociSpecFileName)
-
-	logger.Infof("Using OCI specification file path: %v", OCISpecFilePath)
-
-	return OCISpecFilePath, nil
+	)
 }

cmd/nvidia-container-runtime/runtime_factory_test.go

@@ -17,10 +17,8 @@
 package main
 
 import (
-	"path/filepath"
 	"testing"
 
-	testlog "github.com/sirupsen/logrus/hooks/test"
 	"github.com/stretchr/testify/require"
 )
 
@@ -30,163 +28,3 @@ func TestConstructor(t *testing.T) {
 	require.NoError(t, err)
 	require.NotNil(t, shim)
 }
-
-func TestGetBundlePath(t *testing.T) {
-	type expected struct {
-		bundle  string
-		isError bool
-	}
-	testCases := []struct {
-		argv     []string
-		expected expected
-	}{
-		{
-			argv: []string{},
-		},
-		{
-			argv: []string{"create"},
-		},
-		{
-			argv: []string{"--bundle"},
-			expected: expected{
-				isError: true,
-			},
-		},
-		{
-			argv: []string{"-b"},
-			expected: expected{
-				isError: true,
-			},
-		},
-		{
-			argv: []string{"--bundle", "/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"--not-bundle", "/foo/bar"},
-		},
-		{
-			argv: []string{"--"},
-		},
-		{
-			argv: []string{"-bundle", "/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"--bundle=/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"-b=/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"-b=/foo/=bar"},
-			expected: expected{
-				bundle: "/foo/=bar",
-			},
-		},
-		{
-			argv: []string{"-b", "/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"create", "-b", "/foo/bar"},
-			expected: expected{
-				bundle: "/foo/bar",
-			},
-		},
-		{
-			argv: []string{"-b", "create", "create"},
-			expected: expected{
-				bundle: "create",
-			},
-		},
-		{
-			argv: []string{"-b=create", "create"},
-			expected: expected{
-				bundle: "create",
-			},
-		},
-		{
-			argv: []string{"-b", "create"},
-			expected: expected{
-				bundle: "create",
-			},
-		},
-	}
-
-	for i, tc := range testCases {
-		bundle, err := getBundlePath(tc.argv)
-
-		if tc.expected.isError {
-			require.Errorf(t, err, "%d: %v", i, tc)
-		} else {
-			require.NoErrorf(t, err, "%d: %v", i, tc)
-		}
-
-		require.Equalf(t, tc.expected.bundle, bundle, "%d: %v", i, tc)
-	}
-}
-
-func TestFindRunc(t *testing.T) {
-	testLogger, _ := testlog.NewNullLogger()
-	logger.Logger = testLogger
-
-	runcPath, err := findRunc()
-	require.NoError(t, err)
-	require.Equal(t, filepath.Join(cfg.binPath, runcExecutableName), runcPath)
-}
-
-func TestFindRuntime(t *testing.T) {
-	testLogger, _ := testlog.NewNullLogger()
-	logger.Logger = testLogger
-
-	testCases := []struct {
-		candidates   []string
-		expectedPath string
-	}{
-		{
-			candidates: []string{},
-		},
-		{
-			candidates: []string{"not-runc"},
-		},
-		{
-			candidates: []string{"not-runc", "also-not-runc"},
-		},
-		{
-			candidates:   []string{runcExecutableName},
-			expectedPath: filepath.Join(cfg.binPath, runcExecutableName),
-		},
-		{
-			candidates:   []string{runcExecutableName, "not-runc"},
-			expectedPath: filepath.Join(cfg.binPath, runcExecutableName),
-		},
-		{
-			candidates:   []string{"not-runc", runcExecutableName},
-			expectedPath: filepath.Join(cfg.binPath, runcExecutableName),
-		},
-	}
-
-	for i, tc := range testCases {
-		runcPath, err := findRuntime(tc.candidates)
-		if tc.expectedPath == "" {
-			require.Error(t, err, "%d: %v", i, tc)
-		} else {
-			require.NoError(t, err, "%d: %v", i, tc)
-		}
-		require.Equal(t, tc.expectedPath, runcPath, "%d: %v", i, tc)
-	}
-
-}

docker/Dockerfile.debian

@@ -61,6 +61,8 @@ WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
 RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
+    dch --changelog debian/changelog -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \

docker/Dockerfile.ubuntu

@@ -54,6 +54,8 @@ WORKDIR $DIST_DIR
 COPY packaging/debian ./debian
 
 RUN sed -i "s;@VERSION@;${REVISION};" debian/changelog && \
+    dch --changelog debian/changelog --append "Bump libnvidia-container dependency to ${REVISION}}" && \
+    dch --changelog debian/changelog -r "" && \
     if [ "$REVISION" != "$(dpkg-parsechangelog --show-field=Version)" ]; then exit 1; fi
 
 CMD export DISTRIB="$(lsb_release -cs)" && \

go.mod

@@ -4,10 +4,14 @@ go 1.14
 
 require (
 	github.com/BurntSushi/toml v0.3.1
-	github.com/opencontainers/runtime-spec v1.0.2
+	github.com/containerd/containerd v1.5.7
+	github.com/containers/podman/v2 v2.2.1
+	github.com/opencontainers/runtime-spec v1.0.3-0.20211101234015-a3c33d663ebc
 	github.com/pelletier/go-toml v1.9.3
 	github.com/sirupsen/logrus v1.8.1
 	github.com/stretchr/testify v1.7.0
 	github.com/tsaikd/KDGoLib v0.0.0-20191001134900-7f3cf518e07d
+	github.com/urfave/cli/v2 v2.3.0
 	golang.org/x/mod v0.3.0
+	golang.org/x/sys v0.0.0-20210426230700-d19ff857e887
 )

internal/oci/args.go

@@ -0,0 +1,115 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+)
+
+const (
+	specFileName = "config.json"
+)
+
+// GetBundleDir returns the bundle directory or default depending on the
+// supplied command line arguments.
+func GetBundleDir(args []string) (string, error) {
+	bundleDir, err := GetBundleDirFromArgs(args)
+	if err != nil {
+		return "", fmt.Errorf("error getting bundle dir from args: %v", err)
+	}
+
+	return bundleDir, nil
+}
+
+// GetBundleDirFromArgs checks the specified slice of strings (argv) for a 'bundle' flag as allowed by runc.
+// The following are supported:
+// --bundle{{SEP}}BUNDLE_PATH
+// -bundle{{SEP}}BUNDLE_PATH
+// -b{{SEP}}BUNDLE_PATH
+// where {{SEP}} is either ' ' or '='
+func GetBundleDirFromArgs(args []string) (string, error) {
+	var bundleDir string
+
+	for i := 0; i < len(args); i++ {
+		param := args[i]
+
+		parts := strings.SplitN(param, "=", 2)
+		if !IsBundleFlag(parts[0]) {
+			continue
+		}
+
+		// The flag has the format --bundle=/path
+		if len(parts) == 2 {
+			bundleDir = parts[1]
+			continue
+		}
+
+		// The flag has the format --bundle /path
+		if i+1 < len(args) {
+			bundleDir = args[i+1]
+			i++
+			continue
+		}
+
+		// --bundle / -b was the last element of args
+		return "", fmt.Errorf("bundle option requires an argument")
+	}
+
+	return bundleDir, nil
+}
+
+// GetSpecFilePath returns the expected path to the OCI specification file for the given
+// bundle directory.
+func GetSpecFilePath(bundleDir string) string {
+	specFilePath := filepath.Join(bundleDir, specFileName)
+	return specFilePath
+}
+
+// IsBundleFlag is a helper function that checks wither the specified argument represents
+// a bundle flag (--bundle or -b)
+func IsBundleFlag(arg string) bool {
+	if !strings.HasPrefix(arg, "-") {
+		return false
+	}
+
+	trimmed := strings.TrimLeft(arg, "-")
+	return trimmed == "b" || trimmed == "bundle"
+}
+
+// HasCreateSubcommand checks the supplied arguments for a 'create' subcommand
+func HasCreateSubcommand(args []string) bool {
+	var previousWasBundle bool
+	for _, a := range args {
+		// We check for '--bundle create' explicitly to ensure that we
+		// don't inadvertently trigger a modification if the bundle directory
+		// is specified as `create`
+		if !previousWasBundle && IsBundleFlag(a) {
+			previousWasBundle = true
+			continue
+		}
+
+		if !previousWasBundle && a == "create" {
+			return true
+		}
+
+		previousWasBundle = false
+	}
+
+	return false
+}

internal/oci/args_test.go

@@ -0,0 +1,184 @@
+package oci
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestGetBundleDir(t *testing.T) {
+	type expected struct {
+		bundle  string
+		isError bool
+	}
+	testCases := []struct {
+		argv     []string
+		expected expected
+	}{
+		{
+			argv: []string{},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"create"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"--bundle"},
+			expected: expected{
+				isError: true,
+			},
+		},
+		{
+			argv: []string{"-b"},
+			expected: expected{
+				isError: true,
+			},
+		},
+		{
+			argv: []string{"--bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"--not-bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"--"},
+			expected: expected{
+				bundle: "",
+			},
+		},
+		{
+			argv: []string{"-bundle", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"--bundle=/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b=/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b=/foo/=bar"},
+			expected: expected{
+				bundle: "/foo/=bar",
+			},
+		},
+		{
+			argv: []string{"-b", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"create", "-b", "/foo/bar"},
+			expected: expected{
+				bundle: "/foo/bar",
+			},
+		},
+		{
+			argv: []string{"-b", "create", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+		{
+			argv: []string{"-b=create", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+		{
+			argv: []string{"-b", "create"},
+			expected: expected{
+				bundle: "create",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		bundle, err := GetBundleDir(tc.argv)
+
+		if tc.expected.isError {
+			require.Errorf(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoErrorf(t, err, "%d: %v", i, tc)
+		}
+
+		require.Equalf(t, tc.expected.bundle, bundle, "%d: %v", i, tc)
+	}
+}
+
+func TestGetSpecFilePathAppendsFilename(t *testing.T) {
+	testCases := []struct {
+		bundleDir string
+		expected  string
+	}{
+		{
+			bundleDir: "",
+			expected:  "config.json",
+		},
+		{
+			bundleDir: "/not/empty/",
+			expected:  "/not/empty/config.json",
+		},
+		{
+			bundleDir: "not/absolute",
+			expected:  "not/absolute/config.json",
+		},
+	}
+
+	for i, tc := range testCases {
+		specPath := GetSpecFilePath(tc.bundleDir)
+
+		require.Equalf(t, tc.expected, specPath, "%d: %v", i, tc)
+	}
+}
+
+func TestHasCreateSubcommand(t *testing.T) {
+	testCases := []struct {
+		args         []string
+		shouldModify bool
+	}{
+		{
+			shouldModify: false,
+		},
+		{
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+		{
+			args:         []string{"--bundle=create"},
+			shouldModify: false,
+		},
+		{
+			args:         []string{"--bundle", "create"},
+			shouldModify: false,
+		},
+		{
+			args:         []string{"create"},
+			shouldModify: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		require.Equal(t, tc.shouldModify, HasCreateSubcommand(tc.args), "%d: %v", i, tc)
+	}
+}

internal/oci/runtime.go

@@ -16,6 +16,8 @@
 
 package oci
 
+//go:generate moq -stub -out runtime_mock.go . Runtime
+
 // Runtime is an interface for a runtime shim. The Exec method accepts a list
 // of command line arguments, and returns an error / nil.
 type Runtime interface {

internal/oci/runtime_exec.go

@@ -1,79 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-
-package oci
-
-import (
-	"fmt"
-	"os"
-	"syscall"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// SyscallExecRuntime wraps the path that a binary and defines the semanitcs for how to exec into it.
-// This can be used to wrap an OCI-compliant low-level runtime binary, allowing it to be used through the
-// Runtime internface.
-type SyscallExecRuntime struct {
-	logger *log.Logger
-	path   string
-	// exec is used for testing. This defaults to syscall.Exec
-	exec func(argv0 string, argv []string, envv []string) error
-}
-
-var _ Runtime = (*SyscallExecRuntime)(nil)
-
-// NewSyscallExecRuntime creates a SyscallExecRuntime for the specified path with the standard logger
-func NewSyscallExecRuntime(path string) (Runtime, error) {
-	return NewSyscallExecRuntimeWithLogger(log.StandardLogger(), path)
-}
-
-// NewSyscallExecRuntimeWithLogger creates a SyscallExecRuntime for the specified logger and path
-func NewSyscallExecRuntimeWithLogger(logger *log.Logger, path string) (Runtime, error) {
-	info, err := os.Stat(path)
-	if err != nil {
-		return nil, fmt.Errorf("invalid path '%v': %v", path, err)
-	}
-	if info.IsDir() || info.Mode()&0111 == 0 {
-		return nil, fmt.Errorf("specified path '%v' is not an executable file", path)
-	}
-
-	shim := SyscallExecRuntime{
-		logger: logger,
-		path:   path,
-		exec:   syscall.Exec,
-	}
-
-	return &shim, nil
-}
-
-// Exec exces into the binary at the path from the SyscallExecRuntime struct, passing it the supplied arguments
-// after ensuring that the first argument is the path of the target binary.
-func (s SyscallExecRuntime) Exec(args []string) error {
-	runtimeArgs := []string{s.path}
-	if len(args) > 1 {
-		runtimeArgs = append(runtimeArgs, args[1:]...)
-	}
-
-	err := s.exec(s.path, runtimeArgs, os.Environ())
-	if err != nil {
-		return fmt.Errorf("could not exec '%v': %v", s.path, err)
-	}
-
-	// syscall.Exec is not expected to return. This is an error state regardless of whether
-	// err is nil or not.
-	return fmt.Errorf("unexpected return from exec '%v'", s.path)
-}

internal/oci/runtime_exec_test.go

@@ -1,100 +0,0 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
-package oci
-
-import (
-	"fmt"
-	"strings"
-	"testing"
-
-	testlog "github.com/sirupsen/logrus/hooks/test"
-	"github.com/stretchr/testify/require"
-)
-
-func TestSyscallExecConstructor(t *testing.T) {
-	r, err := NewSyscallExecRuntime("////an/invalid/path")
-	require.Error(t, err)
-	require.Nil(t, r)
-
-	r, err = NewSyscallExecRuntime("/tmp")
-	require.Error(t, err)
-	require.Nil(t, r)
-
-	r, err = NewSyscallExecRuntime("/dev/null")
-	require.Error(t, err)
-	require.Nil(t, r)
-
-	r, err = NewSyscallExecRuntime("/bin/sh")
-	require.NoError(t, err)
-
-	f, ok := r.(*SyscallExecRuntime)
-	require.True(t, ok)
-
-	require.Equal(t, "/bin/sh", f.path)
-}
-
-func TestSyscallExecForwardsArgs(t *testing.T) {
-	logger, _ := testlog.NewNullLogger()
-	f := SyscallExecRuntime{
-		logger: logger,
-		path:   "runtime",
-	}
-
-	testCases := []struct {
-		returnError error
-		args        []string
-		errorPrefix string
-	}{
-		{
-			returnError: nil,
-			errorPrefix: "unexpected return from exec",
-		},
-		{
-			returnError: fmt.Errorf("error from exec"),
-			errorPrefix: "could not exec",
-		},
-		{
-			returnError: nil,
-			args:        []string{"otherargv0"},
-			errorPrefix: "unexpected return from exec",
-		},
-		{
-			returnError: nil,
-			args:        []string{"otherargv0", "arg1", "arg2", "arg3"},
-			errorPrefix: "unexpected return from exec",
-		},
-	}
-
-	for i, tc := range testCases {
-		execMock := WithMockExec(f, tc.returnError)
-
-		err := execMock.Exec(tc.args)
-
-		require.Errorf(t, err, "%d: %v", i, tc)
-		require.Truef(t, strings.HasPrefix(err.Error(), tc.errorPrefix), "%d: %v", i, tc)
-		if tc.returnError != nil {
-			require.Truef(t, strings.HasSuffix(err.Error(), tc.returnError.Error()), "%d: %v", i, tc)
-		}
-
-		require.Equalf(t, f.path, execMock.argv0, "%d: %v", i, tc)
-		require.Equalf(t, f.path, execMock.argv[0], "%d: %v", i, tc)
-
-		require.LessOrEqualf(t, len(tc.args), len(execMock.argv), "%d: %v", i, tc)
-		if len(tc.args) > 1 {
-			require.Equalf(t, tc.args[1:], execMock.argv[1:], "%d: %v", i, tc)
-		}
-	}
-}

internal/oci/runtime_low_level.go

@@ -0,0 +1,61 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os/exec"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// NewLowLevelRuntime creates a Runtime that wraps a low-level runtime executable.
+// The executable specified is taken from the list of supplied candidates, with the first match
+// present in the PATH being selected.
+func NewLowLevelRuntime(candidates ...string) (Runtime, error) {
+	return NewLowLevelRuntimeWithLogger(log.StandardLogger(), candidates...)
+}
+
+// NewLowLevelRuntimeWithLogger creates a Runtime as with NewLowLevelRuntime using the specified logger.
+func NewLowLevelRuntimeWithLogger(logger *log.Logger, candidates ...string) (Runtime, error) {
+	runtimePath, err := findRuntime(candidates)
+	if err != nil {
+		return nil, fmt.Errorf("error locating runtime: %v", err)
+	}
+
+	return NewRuntimeForPathWithLogger(logger, runtimePath)
+}
+
+// findRuntime checks elements in a list of supplied candidates for a matching executable in the PATH.
+// The absolute path to the first match is returned.
+func findRuntime(candidates []string) (string, error) {
+	if len(candidates) == 0 {
+		return "", fmt.Errorf("at least one runtime candidate must be specified")
+	}
+
+	for _, candidate := range candidates {
+		log.Infof("Looking for runtime binary '%v'", candidate)
+		runcPath, err := exec.LookPath(candidate)
+		if err == nil {
+			log.Infof("Found runtime binary '%v'", runcPath)
+			return runcPath, nil
+		}
+		log.Warnf("Runtime binary '%v' not found: %v", candidate, err)
+	}
+
+	return "", fmt.Errorf("no runtime binary found from candidate list: %v", candidates)
+}

internal/oci/runtime_mock.go

@@ -1,49 +1,76 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
 
 package oci
 
-// MockExecRuntime wraps a SyscallExecRuntime, intercepting the exec call for testing
-type MockExecRuntime struct {
-	SyscallExecRuntime
-	execMock
-}
+import (
+	"sync"
+)
 
-// WithMockExec wraps a specified SyscallExecRuntime with a mocked exec function for testing
-func WithMockExec(e SyscallExecRuntime, execResult error) *MockExecRuntime {
-	m := MockExecRuntime{
-		SyscallExecRuntime: e,
-		execMock:           execMock{result: execResult},
+// Ensure, that RuntimeMock does implement Runtime.
+// If this is not the case, regenerate this file with moq.
+var _ Runtime = &RuntimeMock{}
+
+// RuntimeMock is a mock implementation of Runtime.
+//
+// 	func TestSomethingThatUsesRuntime(t *testing.T) {
+//
+// 		// make and configure a mocked Runtime
+// 		mockedRuntime := &RuntimeMock{
+// 			ExecFunc: func(strings []string) error {
+// 				panic("mock out the Exec method")
+// 			},
+// 		}
+//
+// 		// use mockedRuntime in code that requires Runtime
+// 		// and then make assertions.
+//
+// 	}
+type RuntimeMock struct {
+	// ExecFunc mocks the Exec method.
+	ExecFunc func(strings []string) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Exec holds details about calls to the Exec method.
+		Exec []struct {
+			// Strings is the strings argument value.
+			Strings []string
+		}
 	}
-	// overrdie the exec function to the mocked exec function.
-	m.SyscallExecRuntime.exec = m.execMock.exec
-	return &m
+	lockExec sync.RWMutex
 }
 
-type execMock struct {
-	argv0  string
-	argv   []string
-	envv   []string
-	result error
+// Exec calls ExecFunc.
+func (mock *RuntimeMock) Exec(strings []string) error {
+	callInfo := struct {
+		Strings []string
+	}{
+		Strings: strings,
+	}
+	mock.lockExec.Lock()
+	mock.calls.Exec = append(mock.calls.Exec, callInfo)
+	mock.lockExec.Unlock()
+	if mock.ExecFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.ExecFunc(strings)
 }
 
-func (m *execMock) exec(argv0 string, argv []string, envv []string) error {
-	m.argv0 = argv0
-	m.argv = argv
-	m.envv = envv
-
-	return m.result
+// ExecCalls gets all the calls that were made to Exec.
+// Check the length with:
+//     len(mockedRuntime.ExecCalls())
+func (mock *RuntimeMock) ExecCalls() []struct {
+	Strings []string
+} {
+	var calls []struct {
+		Strings []string
+	}
+	mock.lockExec.RLock()
+	calls = mock.calls.Exec
+	mock.lockExec.RUnlock()
+	return calls
 }

internal/oci/runtime_path.go

@@ -0,0 +1,70 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+// pathRuntime wraps the path that a binary and defines the semanitcs for how to exec into it.
+// This can be used to wrap an OCI-compliant low-level runtime binary, allowing it to be used through the
+// Runtime internface.
+type pathRuntime struct {
+	logger      *log.Logger
+	path        string
+	execRuntime Runtime
+}
+
+var _ Runtime = (*pathRuntime)(nil)
+
+// NewRuntimeForPath creates a Runtime for the specified path with the standard logger
+func NewRuntimeForPath(path string) (Runtime, error) {
+	return NewRuntimeForPathWithLogger(log.StandardLogger(), path)
+}
+
+// NewRuntimeForPathWithLogger creates a Runtime for the specified logger and path
+func NewRuntimeForPathWithLogger(logger *log.Logger, path string) (Runtime, error) {
+	info, err := os.Stat(path)
+	if err != nil {
+		return nil, fmt.Errorf("invalid path '%v': %v", path, err)
+	}
+	if info.IsDir() || info.Mode()&0111 == 0 {
+		return nil, fmt.Errorf("specified path '%v' is not an executable file", path)
+	}
+
+	shim := pathRuntime{
+		logger:      logger,
+		path:        path,
+		execRuntime: syscallExec{},
+	}
+
+	return &shim, nil
+}
+
+// Exec exces into the binary at the path from the pathRuntime struct, passing it the supplied arguments
+// after ensuring that the first argument is the path of the target binary.
+func (s pathRuntime) Exec(args []string) error {
+	runtimeArgs := []string{s.path}
+	if len(args) > 1 {
+		runtimeArgs = append(runtimeArgs, args[1:]...)
+	}
+
+	return s.execRuntime.Exec(runtimeArgs)
+}

internal/oci/runtime_path_test.go

@@ -0,0 +1,97 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+package oci
+
+import (
+	"fmt"
+	"testing"
+
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPathRuntimeConstructor(t *testing.T) {
+	r, err := NewRuntimeForPath("////an/invalid/path")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath("/tmp")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath("/dev/null")
+	require.Error(t, err)
+	require.Nil(t, r)
+
+	r, err = NewRuntimeForPath("/bin/sh")
+	require.NoError(t, err)
+
+	f, ok := r.(*pathRuntime)
+	require.True(t, ok)
+
+	require.Equal(t, "/bin/sh", f.path)
+}
+
+func TestPathRuntimeForwardsArgs(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testCases := []struct {
+		execRuntimeError error
+		args             []string
+	}{
+		{},
+		{
+			args: []string{"shouldBeReplaced"},
+		},
+		{
+			args: []string{"shouldBeReplaced", "arg1"},
+		},
+		{
+			execRuntimeError: fmt.Errorf("exec error"),
+		},
+	}
+
+	for _, tc := range testCases {
+		mockedRuntime := &RuntimeMock{
+			ExecFunc: func(strings []string) error {
+				return tc.execRuntimeError
+			},
+		}
+		r := pathRuntime{
+			logger:      logger,
+			path:        "runtime",
+			execRuntime: mockedRuntime,
+		}
+		err := r.Exec(tc.args)
+
+		require.ErrorIs(t, err, tc.execRuntimeError)
+
+		calls := mockedRuntime.ExecCalls()
+		require.Len(t, calls, 1)
+
+		numArgs := len(tc.args)
+		if numArgs == 0 {
+			numArgs = 1
+		}
+
+		require.Len(t, calls[0].Strings, numArgs)
+		require.Equal(t, "runtime", calls[0].Strings[0])
+
+		if numArgs > 1 {
+			require.EqualValues(t, tc.args[1:], calls[0].Strings[1:])
+		}
+	}
+}

internal/oci/runtime_syscall_exec.go

@@ -0,0 +1,38 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"fmt"
+	"os"
+	"syscall"
+)
+
+type syscallExec struct{}
+
+var _ Runtime = (*syscallExec)(nil)
+
+func (r syscallExec) Exec(args []string) error {
+	err := syscall.Exec(args[0], args, os.Environ())
+	if err != nil {
+		return fmt.Errorf("could not exec '%v': %v", args[0], err)
+	}
+
+	// syscall.Exec is not expected to return. This is an error state regardless of whether
+	// err is nil or not.
+	return fmt.Errorf("unexpected return from exec '%v'", args[0])
+}

internal/oci/spec.go

@@ -17,10 +17,6 @@
 package oci
 
 import (
-	"encoding/json"
-	"fmt"
-	"os"
-
 	oci "github.com/opencontainers/runtime-spec/specs-go"
 )
 
@@ -28,75 +24,12 @@ import (
 // error. The intention is that the function would modify the spec in-place.
 type SpecModifier func(*oci.Spec) error
 
+//go:generate moq -stub -out spec_mock.go . Spec
+
 // Spec defines the operations to be performed on an OCI specification
 type Spec interface {
 	Load() error
 	Flush() error
 	Modify(SpecModifier) error
-}
-
-type fileSpec struct {
-	*oci.Spec
-	path string
-}
-
-var _ Spec = (*fileSpec)(nil)
-
-// NewSpecFromFile creates an object that encapsulates a file-backed OCI spec.
-// This can be used to read from the file, modify the spec, and write to the
-// same file.
-func NewSpecFromFile(filepath string) Spec {
-	oci := fileSpec{
-		path: filepath,
-	}
-
-	return &oci
-}
-
-// Load reads the contents of an OCI spec from file to be referenced internally.
-// The file is opened "read-only"
-func (s *fileSpec) Load() error {
-	specFile, err := os.Open(s.path)
-	if err != nil {
-		return fmt.Errorf("error opening OCI specification file: %v", err)
-	}
-	defer specFile.Close()
-
-	decoder := json.NewDecoder(specFile)
-
-	var spec oci.Spec
-	err = decoder.Decode(&spec)
-	if err != nil {
-		return fmt.Errorf("error reading OCI specification from file: %v", err)
-	}
-
-	s.Spec = &spec
-	return nil
-}
-
-// Modify applies the specified SpecModifier to the stored OCI specification.
-func (s *fileSpec) Modify(f SpecModifier) error {
-	if s.Spec == nil {
-		return fmt.Errorf("no spec loaded for modification")
-	}
-	return f(s.Spec)
-}
-
-// Flush writes the stored OCI specification to the filepath specifed by the path member.
-// The file is truncated upon opening, overwriting any existing contents.
-func (s fileSpec) Flush() error {
-	specFile, err := os.Create(s.path)
-	if err != nil {
-		return fmt.Errorf("error opening OCI specification file: %v", err)
-	}
-	defer specFile.Close()
-
-	encoder := json.NewEncoder(specFile)
-
-	err = encoder.Encode(s.Spec)
-	if err != nil {
-		return fmt.Errorf("error writing OCI specification to file: %v", err)
-	}
-
-	return nil
+	LookupEnv(string) (string, bool)
 }

internal/oci/spec_file.go

@@ -0,0 +1,153 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strings"
+
+	oci "github.com/opencontainers/runtime-spec/specs-go"
+)
+
+type fileSpec struct {
+	*oci.Spec
+	path string
+}
+
+var _ Spec = (*fileSpec)(nil)
+
+// NewSpecFromArgs creates fileSpec based on the command line arguments passed to the
+// application
+func NewSpecFromArgs(args []string) (Spec, string, error) {
+	bundleDir, err := GetBundleDir(args)
+	if err != nil {
+		return nil, "", fmt.Errorf("error getting bundle directory: %v", err)
+	}
+
+	ociSpecPath := GetSpecFilePath(bundleDir)
+
+	ociSpec := NewSpecFromFile(ociSpecPath)
+
+	return ociSpec, bundleDir, nil
+}
+
+// NewSpecFromFile creates an object that encapsulates a file-backed OCI spec.
+// This can be used to read from the file, modify the spec, and write to the
+// same file.
+func NewSpecFromFile(filepath string) Spec {
+	oci := fileSpec{
+		path: filepath,
+	}
+
+	return &oci
+}
+
+// Load reads the contents of an OCI spec from file to be referenced internally.
+// The file is opened "read-only"
+func (s *fileSpec) Load() error {
+	specFile, err := os.Open(s.path)
+	if err != nil {
+		return fmt.Errorf("error opening OCI specification file: %v", err)
+	}
+	defer specFile.Close()
+
+	return s.loadFrom(specFile)
+}
+
+// loadFrom reads the contents of the OCI spec from the specified io.Reader.
+func (s *fileSpec) loadFrom(reader io.Reader) error {
+	decoder := json.NewDecoder(reader)
+
+	var spec oci.Spec
+	err := decoder.Decode(&spec)
+	if err != nil {
+		return fmt.Errorf("error reading OCI specification: %v", err)
+	}
+
+	s.Spec = &spec
+	return nil
+}
+
+// Modify applies the specified SpecModifier to the stored OCI specification.
+func (s *fileSpec) Modify(f SpecModifier) error {
+	if s.Spec == nil {
+		return fmt.Errorf("no spec loaded for modification")
+	}
+	return f(s.Spec)
+}
+
+// Flush writes the stored OCI specification to the filepath specifed by the path member.
+// The file is truncated upon opening, overwriting any existing contents.
+func (s fileSpec) Flush() error {
+	if s.Spec == nil {
+		return fmt.Errorf("no OCI specification loaded")
+	}
+
+	specFile, err := os.Create(s.path)
+	if err != nil {
+		return fmt.Errorf("error opening OCI specification file: %v", err)
+	}
+	defer specFile.Close()
+
+	return s.flushTo(specFile)
+}
+
+// flushTo writes the stored OCI specification to the specified io.Writer.
+func (s fileSpec) flushTo(writer io.Writer) error {
+	if s.Spec == nil {
+		return nil
+	}
+	encoder := json.NewEncoder(writer)
+
+	err := encoder.Encode(s.Spec)
+	if err != nil {
+		return fmt.Errorf("error writing OCI specification: %v", err)
+	}
+
+	return nil
+}
+
+// LookupEnv mirrors os.LookupEnv for the OCI specification. It
+// retrieves the value of the environment variable named
+// by the key. If the variable is present in the environment the
+// value (which may be empty) is returned and the boolean is true.
+// Otherwise the returned value will be empty and the boolean will
+// be false.
+func (s fileSpec) LookupEnv(key string) (string, bool) {
+	if s.Spec == nil || s.Spec.Process == nil {
+		return "", false
+	}
+
+	for _, env := range s.Spec.Process.Env {
+		if !strings.HasPrefix(env, key) {
+			continue
+		}
+
+		parts := strings.SplitN(env, "=", 2)
+		if parts[0] == key {
+			if len(parts) < 2 {
+				return "", true
+			}
+			return parts[1], true
+		}
+	}
+
+	return "", false
+}

internal/oci/spec_file_test.go

@@ -0,0 +1,252 @@
+/*
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package oci
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLookupEnv(t *testing.T) {
+	const envName = "TEST_ENV"
+	testCases := []struct {
+		spec          *specs.Spec
+		expectedValue string
+		expectedExits bool
+	}{
+		{
+			// nil spec
+			spec:          nil,
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// nil process
+			spec:          &specs.Spec{},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// nil env
+			spec: &specs.Spec{
+				Process: &specs.Process{},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// empty env
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// different env set
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"SOMETHING_ELSE=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// same prefix
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV_BUT_NOT=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// same suffix
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"NOT_TEST_ENV=foo"}},
+			},
+			expectedValue: "",
+			expectedExits: false,
+		},
+		{
+			// set blank
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV="}},
+			},
+			expectedValue: "",
+			expectedExits: true,
+		},
+		{
+			// set no-equals
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV"}},
+			},
+			expectedValue: "",
+			expectedExits: true,
+		},
+		{
+			// set value
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV=something"}},
+			},
+			expectedValue: "something",
+			expectedExits: true,
+		},
+		{
+			// set with equals
+			spec: &specs.Spec{
+				Process: &specs.Process{Env: []string{"TEST_ENV=something=somethingelse"}},
+			},
+			expectedValue: "something=somethingelse",
+			expectedExits: true,
+		},
+	}
+
+	for i, tc := range testCases {
+		spec := fileSpec{
+			Spec: tc.spec,
+		}
+
+		value, exists := spec.LookupEnv(envName)
+
+		require.Equal(t, tc.expectedValue, value, "%d: %v", i, tc)
+		require.Equal(t, tc.expectedExits, exists, "%d: %v", i, tc)
+	}
+}
+
+func TestLoadFrom(t *testing.T) {
+	testCases := []struct {
+		contents []byte
+		isError  bool
+		spec     *specs.Spec
+	}{
+		{
+			contents: []byte{},
+			isError:  true,
+		},
+		{
+			contents: []byte("{}"),
+			isError:  false,
+			spec:     &specs.Spec{},
+		},
+	}
+
+	for i, tc := range testCases {
+		spec := fileSpec{}
+		err := spec.loadFrom(bytes.NewReader(tc.contents))
+
+		if tc.isError {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+
+		if tc.spec == nil {
+			require.Nil(t, spec.Spec, "%d: %v", i, tc)
+		} else {
+			require.EqualValues(t, tc.spec, spec.Spec, "%d: %v", i, tc)
+		}
+	}
+}
+
+func TestFlushTo(t *testing.T) {
+	testCases := []struct {
+		isError  bool
+		spec     *specs.Spec
+		contents string
+	}{
+		{
+			spec: nil,
+		},
+		{
+			spec:     &specs.Spec{},
+			contents: "{\"ociVersion\":\"\"}\n",
+		},
+	}
+
+	for i, tc := range testCases {
+		buffer := bytes.Buffer{}
+
+		spec := fileSpec{Spec: tc.spec}
+		err := spec.flushTo(&buffer)
+
+		if tc.isError {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+		}
+
+		require.EqualValues(t, tc.contents, buffer.String(), "%d: %v", i, tc)
+	}
+
+	// Add a simple test for a writer that returns an error when writing
+	spec := fileSpec{Spec: &specs.Spec{}}
+	err := spec.flushTo(errorWriter{})
+	require.Error(t, err)
+}
+
+func TestModify(t *testing.T) {
+
+	testCases := []struct {
+		spec          *specs.Spec
+		modifierError error
+	}{
+		{
+			spec: nil,
+		},
+		{
+			spec: &specs.Spec{},
+		},
+		{
+			spec:          &specs.Spec{},
+			modifierError: fmt.Errorf("error in modifier"),
+		},
+	}
+
+	for i, tc := range testCases {
+		spec := fileSpec{Spec: tc.spec}
+
+		modifier := func(spec *specs.Spec) error {
+			if tc.modifierError == nil {
+				spec.Version = "updated"
+			}
+			return tc.modifierError
+		}
+
+		err := spec.Modify(modifier)
+
+		if tc.spec == nil {
+			require.Error(t, err, "%d: %v", i, tc)
+		} else if tc.modifierError != nil {
+			require.EqualError(t, err, tc.modifierError.Error(), "%d: %v", i, tc)
+			require.EqualValues(t, &specs.Spec{}, spec.Spec, "%d: %v", i, tc)
+		} else {
+			require.NoError(t, err, "%d: %v", i, tc)
+			require.Equal(t, "updated", spec.Spec.Version, "%d: %v", i, tc)
+		}
+	}
+}
+
+// errorWriter implements the io.Writer interface, always returning an error when
+// writing.
+type errorWriter struct{}
+
+func (e errorWriter) Write([]byte) (int, error) {
+	return 0, fmt.Errorf("error writing")
+}

internal/oci/spec_mock.go

@@ -1,70 +1,201 @@
-/*
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-*/
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
 
 package oci
 
 import (
-	oci "github.com/opencontainers/runtime-spec/specs-go"
+	"sync"
 )
 
-// MockSpec provides a simple mock for an OCI spec to be used in testing.
-// It also implements the SpecModifier interface.
-type MockSpec struct {
-	*oci.Spec
-	MockLoad   mockFunc
-	MockFlush  mockFunc
-	MockModify mockFunc
-}
+// Ensure, that SpecMock does implement Spec.
+// If this is not the case, regenerate this file with moq.
+var _ Spec = &SpecMock{}
 
-var _ Spec = (*MockSpec)(nil)
+// SpecMock is a mock implementation of Spec.
+//
+// 	func TestSomethingThatUsesSpec(t *testing.T) {
+//
+// 		// make and configure a mocked Spec
+// 		mockedSpec := &SpecMock{
+// 			FlushFunc: func() error {
+// 				panic("mock out the Flush method")
+// 			},
+// 			LoadFunc: func() error {
+// 				panic("mock out the Load method")
+// 			},
+// 			LookupEnvFunc: func(s string) (string, bool) {
+// 				panic("mock out the LookupEnv method")
+// 			},
+// 			ModifyFunc: func(specModifier SpecModifier) error {
+// 				panic("mock out the Modify method")
+// 			},
+// 		}
+//
+// 		// use mockedSpec in code that requires Spec
+// 		// and then make assertions.
+//
+// 	}
+type SpecMock struct {
+	// FlushFunc mocks the Flush method.
+	FlushFunc func() error
 
-// NewMockSpec constructs a MockSpec to be used in testing as a Spec
-func NewMockSpec(spec *oci.Spec, flushResult error, modifyResult error) *MockSpec {
-	s := MockSpec{
-		Spec:       spec,
-		MockFlush:  mockFunc{result: flushResult},
-		MockModify: mockFunc{result: modifyResult},
+	// LoadFunc mocks the Load method.
+	LoadFunc func() error
+
+	// LookupEnvFunc mocks the LookupEnv method.
+	LookupEnvFunc func(s string) (string, bool)
+
+	// ModifyFunc mocks the Modify method.
+	ModifyFunc func(specModifier SpecModifier) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Flush holds details about calls to the Flush method.
+		Flush []struct {
+		}
+		// Load holds details about calls to the Load method.
+		Load []struct {
+		}
+		// LookupEnv holds details about calls to the LookupEnv method.
+		LookupEnv []struct {
+			// S is the s argument value.
+			S string
+		}
+		// Modify holds details about calls to the Modify method.
+		Modify []struct {
+			// SpecModifier is the specModifier argument value.
+			SpecModifier SpecModifier
+		}
 	}
-
-	return &s
+	lockFlush     sync.RWMutex
+	lockLoad      sync.RWMutex
+	lockLookupEnv sync.RWMutex
+	lockModify    sync.RWMutex
 }
 
-// Load invokes the mocked Load function to return the predefined error / result
-func (s *MockSpec) Load() error {
-	return s.MockLoad.call()
+// Flush calls FlushFunc.
+func (mock *SpecMock) Flush() error {
+	callInfo := struct {
+	}{}
+	mock.lockFlush.Lock()
+	mock.calls.Flush = append(mock.calls.Flush, callInfo)
+	mock.lockFlush.Unlock()
+	if mock.FlushFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.FlushFunc()
 }
 
-// Flush invokes the mocked Load function to return the predefined error / result
-func (s *MockSpec) Flush() error {
-	return s.MockFlush.call()
+// FlushCalls gets all the calls that were made to Flush.
+// Check the length with:
+//     len(mockedSpec.FlushCalls())
+func (mock *SpecMock) FlushCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockFlush.RLock()
+	calls = mock.calls.Flush
+	mock.lockFlush.RUnlock()
+	return calls
 }
 
-// Modify applies the specified SpecModifier to the spec and invokes the
-// mocked modify function to return the predefined error / result.
-func (s *MockSpec) Modify(f SpecModifier) error {
-	f(s.Spec)
-	return s.MockModify.call()
+// Load calls LoadFunc.
+func (mock *SpecMock) Load() error {
+	callInfo := struct {
+	}{}
+	mock.lockLoad.Lock()
+	mock.calls.Load = append(mock.calls.Load, callInfo)
+	mock.lockLoad.Unlock()
+	if mock.LoadFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.LoadFunc()
 }
 
-type mockFunc struct {
-	Callcount int
-	result    error
+// LoadCalls gets all the calls that were made to Load.
+// Check the length with:
+//     len(mockedSpec.LoadCalls())
+func (mock *SpecMock) LoadCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockLoad.RLock()
+	calls = mock.calls.Load
+	mock.lockLoad.RUnlock()
+	return calls
 }
 
-func (m *mockFunc) call() error {
-	m.Callcount++
-	return m.result
+// LookupEnv calls LookupEnvFunc.
+func (mock *SpecMock) LookupEnv(s string) (string, bool) {
+	callInfo := struct {
+		S string
+	}{
+		S: s,
+	}
+	mock.lockLookupEnv.Lock()
+	mock.calls.LookupEnv = append(mock.calls.LookupEnv, callInfo)
+	mock.lockLookupEnv.Unlock()
+	if mock.LookupEnvFunc == nil {
+		var (
+			sOut string
+			bOut bool
+		)
+		return sOut, bOut
+	}
+	return mock.LookupEnvFunc(s)
+}
+
+// LookupEnvCalls gets all the calls that were made to LookupEnv.
+// Check the length with:
+//     len(mockedSpec.LookupEnvCalls())
+func (mock *SpecMock) LookupEnvCalls() []struct {
+	S string
+} {
+	var calls []struct {
+		S string
+	}
+	mock.lockLookupEnv.RLock()
+	calls = mock.calls.LookupEnv
+	mock.lockLookupEnv.RUnlock()
+	return calls
+}
+
+// Modify calls ModifyFunc.
+func (mock *SpecMock) Modify(specModifier SpecModifier) error {
+	callInfo := struct {
+		SpecModifier SpecModifier
+	}{
+		SpecModifier: specModifier,
+	}
+	mock.lockModify.Lock()
+	mock.calls.Modify = append(mock.calls.Modify, callInfo)
+	mock.lockModify.Unlock()
+	if mock.ModifyFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.ModifyFunc(specModifier)
+}
+
+// ModifyCalls gets all the calls that were made to Modify.
+// Check the length with:
+//     len(mockedSpec.ModifyCalls())
+func (mock *SpecMock) ModifyCalls() []struct {
+	SpecModifier SpecModifier
+} {
+	var calls []struct {
+		SpecModifier SpecModifier
+	}
+	mock.lockModify.RLock()
+	calls = mock.calls.Modify
+	mock.lockModify.RUnlock()
+	return calls
 }

internal/oci/spec_test.go

@@ -0,0 +1,58 @@
+package oci
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMaintainSpec(t *testing.T) {
+	moduleRoot, err := getModuleRoot()
+	require.NoError(t, err)
+
+	files := []string{
+		"config.clone3.json",
+	}
+
+	for _, f := range files {
+		inputSpecPath := filepath.Join(moduleRoot, "test/input", f)
+
+		spec := NewSpecFromFile(inputSpecPath).(*fileSpec)
+
+		spec.Load()
+
+		outputSpecPath := filepath.Join(moduleRoot, "test/output", f)
+		spec.path = outputSpecPath
+		spec.Flush()
+
+		inputContents, err := os.ReadFile(inputSpecPath)
+		require.NoError(t, err)
+
+		outputContents, err := os.ReadFile(outputSpecPath)
+		require.NoError(t, err)
+
+		require.JSONEq(t, string(inputContents), string(outputContents))
+	}
+}
+
+func getModuleRoot() (string, error) {
+	_, filename, _, _ := runtime.Caller(0)
+
+	return hasGoMod(filename)
+}
+
+func hasGoMod(dir string) (string, error) {
+	if dir == "" || dir == "/" {
+		return "", fmt.Errorf("module root not found")
+	}
+
+	_, err := os.Stat(filepath.Join(dir, "go.mod"))
+	if err != nil {
+		return hasGoMod(filepath.Dir(dir))
+	}
+	return dir, nil
+}

packaging/debian/changelog

@@ -1,3 +1,15 @@
+nvidia-container-toolkit (1.6.0~rc.3-1) experimental; urgency=medium
+
+  * Move OCI and command line checks for runtime to internal oci package
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Mon, 15 Nov 2021 13:02:23 +0100
+
+nvidia-container-toolkit (1.6.0~rc.2-1) experimental; urgency=medium
+
+  * Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
+
+ -- NVIDIA CORPORATION <cudatools@nvidia.com>  Fri, 05 Nov 2021 12:24:05 +0200
+
 nvidia-container-toolkit (1.6.0~rc.1-1) experimental; urgency=medium
 
   * Add AARCH64 package for Amazon Linux 2

packaging/rpm/SPECS/nvidia-container-toolkit.spec

@@ -64,6 +64,14 @@ rm -f %{_bindir}/nvidia-container-runtime-hook
 /usr/share/containers/oci/hooks.d/oci-nvidia-hook.json
 
 %changelog
+* Mon Nov 15 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.6.0-0.1.rc.3
+
+- Move OCI and command line checks for runtime to internal oci package
+
+* Fri Nov 05 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.6.0-0.1.rc.2
+
+- Use relative path to OCI specification file (config.json) if bundle path is not specified as an argument to the nvidia-container-runtime
+
 * Mon Sep 06 2021 NVIDIA CORPORATION <cudatools@nvidia.com> 1.6.0-0.1.rc.1
 
 - Add AARCH64 package for Amazon Linux 2

scripts/build-all-components.sh

@@ -55,13 +55,19 @@ make -C ${LIBNVIDIA_CONTAINER_ROOT} -f mk/docker.mk ${TARGET}
 # Build nvidia-container-toolkit
 make -C ${NVIDIA_CONTAINER_TOOLKIT_ROOT} ${TARGET}
 
-# We set the TOOLKIT_VERSION for the nvidia-container-runtime and nvidia-docker targets
-# TODO: This is not yet enabled in the makefiles below
-: ${PREVIOUS_TOOLKIT_VERSION:=1.5.1}
-echo "Using TOOLKIT_VERSION=${PREVIOUS_TOOLKIT_VERSION} as previous nvidia-container-toolkit version"
+if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_VERSION} ]]; then
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+fi
 
+# We set the TOOLKIT_VERSION for the nvidia-container-runtime and nvidia-docker targets
 # Build nvidia-container-runtime
-make -C ${NVIDIA_CONTAINER_RUNTIME_ROOT} TOOLKIT_VERSION=${PREVIOUS_TOOLKIT_VERSION} ${TARGET}
+make -C ${NVIDIA_CONTAINER_RUNTIME_ROOT} \
+    TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
+    TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+        ${TARGET}
 
 # Build nvidia-docker2
-make -C ${NVIDIA_DOCKER_ROOT} TOOLKIT_VERSION=${PREVIOUS_TOOLKIT_VERSION} ${TARGET}
+make -C ${NVIDIA_DOCKER_ROOT} \
+    TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
+    TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+        ${TARGET}

scripts/get-component-versions.sh

@@ -34,29 +34,32 @@ PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
 : ${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}
 
 # Get version for libnvidia-container
-libnvidia_container_version=$(grep "#define NVC_VERSION" ${LIBNVIDIA_CONTAINER_ROOT}/src/nvc.h \
+libnvidia_container_version_tag=$(grep "#define NVC_VERSION" ${LIBNVIDIA_CONTAINER_ROOT}/src/nvc.h \
     | sed -e 's/#define NVC_VERSION[[:space:]]"\(.*\)"/\1/')
 
 # Get version for nvidia-container-toolit
 nvidia_container_toolkit_version=$(grep -m 1 "^LIB_VERSION := " ${NVIDIA_CONTAINER_TOOLKIT_ROOT}/Makefile | sed -e 's/LIB_VERSION :=[[:space:]]\(.*\)[[:space:]]*/\1/')
 nvidia_container_toolkit_tag=$(grep -m 1 "^LIB_TAG .= " ${NVIDIA_CONTAINER_TOOLKIT_ROOT}/Makefile | sed -e 's/LIB_TAG .=[[:space:]]\(.*\)[[:space:]]*/\1/')
-nvidia_container_toolkit_version="${nvidia_container_toolkit_version}${nvidia_container_toolkit_tag:+~${nvidia_container_toolkit_tag}}"
+nvidia_container_toolkit_version_tag="${nvidia_container_toolkit_version}${nvidia_container_toolkit_tag:+~${nvidia_container_toolkit_tag}}"
 
 # Get version for nvidia-container-runtime
 nvidia_container_runtime_version=$(grep -m 1 "^LIB_VERSION := " ${NVIDIA_CONTAINER_RUNTIME_ROOT}/Makefile | sed -e 's/LIB_VERSION :=[[:space:]]\(.*\)[[:space:]]*/\1/')
 nvidia_container_runtime_tag=$(grep -m 1 "^LIB_TAG .= " ${NVIDIA_CONTAINER_RUNTIME_ROOT}/Makefile | sed -e 's/LIB_TAG .=[[:space:]]\(.*\)[[:space:]]*/\1/')
-nvidia_container_runtime_version="${nvidia_container_runtime_version}${nvidia_container_runtime_tag:+~${nvidia_container_runtime_tag}}"
+nvidia_container_runtime_version_tag="${nvidia_container_runtime_version}${nvidia_container_runtime_tag:+~${nvidia_container_runtime_tag}}"
 
 # Get version for nvidia-docker
 nvidia_docker_version=$(grep -m 1 "^LIB_VERSION := " ${NVIDIA_DOCKER_ROOT}/Makefile | sed -e 's/LIB_VERSION :=[[:space:]]\(.*\)[[:space:]]*/\1/')
 nvidia_docker_tag=$(grep -m 1 "^LIB_TAG .= " ${NVIDIA_DOCKER_ROOT}/Makefile | sed -e 's/LIB_TAG .=[[:space:]]\(.*\)[[:space:]]*/\1/')
-nvidia_docker_version="${nvidia_docker_version}${nvidia_docker_tag:+~${nvidia_docker_tag}}"
+nvidia_docker_version_tag="${nvidia_docker_version}${nvidia_docker_tag:+~${nvidia_docker_tag}}"
 
 
-echo "LIBNVIDIA_CONTAINER_VERSION=${libnvidia_container_version}"
+echo "LIBNVIDIA_CONTAINER_VERSION=${libnvidia_container_version_tag}"
 echo "NVIDIA_CONTAINER_TOOLKIT_VERSION=${nvidia_container_toolkit_version}"
-if [[ "${libnvidia_container_version}" != "${nvidia_container_toolkit_version}" ]]; then
+echo "NVIDIA_CONTAINER_TOOLKIT_TAG=${nvidia_container_toolkit_tag}"
+if [[ "${libnvidia_container_version_tag}" != "${nvidia_container_toolkit_version_tag}" ]]; then
     >&2 echo "WARNING: The libnvidia-container and nvidia-container-toolkit versions do not match"
 fi
 echo "NVIDIA_CONTAINER_RUNTIME_VERSION=${nvidia_container_runtime_version}"
+echo "NVIDIA_CONTAINER_RUNTIME_TAG=${nvidia_container_runtime_tag}"
 echo "NVIDIA_DOCKER_VERSION=${nvidia_docker_version}"
+echo "NVIDIA_DOCKER_TAG=${nvidia_docker_tag}"

scripts/release.sh

@@ -52,6 +52,20 @@ else
     targets=${all[@]}
 fi
 
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+export NVIDIA_CONTAINER_TOOLKIT_VERSION
+export NVIDIA_CONTAINER_TOOLKIT_TAG
+
+if [[ "${NVIDIA_CONTAINER_TOOLKIT_TAG}" != "${NVIDIA_CONTAINER_RUNTIME_TAG}" ]]; then
+    echo "ERROR: The nvidia-container-runtime and nvidia-container-toolkit version tags do not match"
+    exit  1
+fi
+
+if [[ "${NVIDIA_CONTAINER_TOOLKIT_TAG}" != "${NVIDIA_DOCKER_TAG}" ]]; then
+    echo "ERROR: The nvidia-docker and nvidia-container-toolkit version tags do not match"
+    exit 1
+fi
+
 for target in ${targets[@]}; do
     ${SCRIPTS_DIR}/build-all-components.sh ${target}
 done

test/container/common.sh

@@ -0,0 +1,117 @@
+#! /bin/bash
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readonly CRIO_HOOKS_DIR="/usr/share/containers/oci/hooks.d"
+readonly CRIO_HOOK_FILENAME="oci-nvidia-hook.json"
+
+# shellcheck disable=SC2015
+[ -t 2 ] && readonly LOG_TTY=1 || readonly LOG_NO_TTY=1
+
+if [ "${LOG_TTY-0}" -eq 1 ] && [ "$(tput colors)" -ge 15 ]; then
+	readonly FMT_BOLD=$(tput bold)
+	readonly FMT_RED=$(tput setaf 1)
+	readonly FMT_YELLOW=$(tput setaf 3)
+	readonly FMT_BLUE=$(tput setaf 12)
+	readonly FMT_CLEAR=$(tput sgr0)
+fi
+
+log() {
+	local -r level="$1"; shift
+	local -r message="$*"
+
+	local fmt_on="${FMT_CLEAR-}"
+	local -r fmt_off="${FMT_CLEAR-}"
+
+	case "${level}" in
+		INFO)  fmt_on="${FMT_BLUE-}" ;;
+		WARN)  fmt_on="${FMT_YELLOW-}" ;;
+		ERROR) fmt_on="${FMT_RED-}" ;;
+	esac
+	printf "%s[%s]%s %b\n" "${fmt_on}" "${level}" "${fmt_off}" "${message}" >&2
+}
+
+with_retry() {
+	local max_attempts="$1"
+	local delay="$2"
+	local count=0
+	local rc
+	shift 2
+
+	while true; do
+		set +e
+		"$@"; rc="$?"
+		set -e
+
+		count="$((count+1))"
+
+		if [[ "${rc}" -eq 0 ]]; then
+			return 0
+		fi
+
+		if [[ "${max_attempts}" -le 0 ]] || [[ "${count}" -lt "${max_attempts}" ]]; then
+			sleep "${delay}"
+		else
+			break
+		fi
+	done
+
+	return 1
+}
+
+testing::setup() {
+	cp -Rp ${basedir}/shared ${shared_dir}
+	mkdir -p "${shared_dir}/etc/containerd"
+	mkdir -p "${shared_dir}/etc/docker"
+	mkdir -p "${shared_dir}/run/docker/containerd"
+	mkdir -p "${shared_dir}/run/nvidia"
+	mkdir -p "${shared_dir}/usr/local/nvidia"
+	mkdir -p "${shared_dir}${CRIO_HOOKS_DIR}"
+}
+
+testing::cleanup() {
+	if [[ "${CLEANUP}" == "false" ]]; then
+		echo "Skipping cleanup: CLEANUP=${CLEANUP}"
+		return 0
+	fi
+	if [[ -e "${shared_dir}" ]]; then
+		docker run --rm \
+			-v "${shared_dir}:/work" \
+			alpine sh -c 'rm -rf /work/*'
+		rmdir "${shared_dir}"
+	fi
+
+	if [[ "${test_cases:-""}" == "" ]]; then
+		echo "No test cases defined. Skipping test case cleanup"
+		return 0
+	fi
+
+	for tc in ${test_cases}; do
+		testing::${tc}::cleanup
+	done
+}
+
+testing::docker_run::toolkit::shell() {
+	docker run --rm --privileged \
+		--entrypoint sh \
+		-v "${shared_dir}/etc/containerd:/etc/containerd" \
+		-v "${shared_dir}/etc/docker:/etc/docker" \
+		-v "${shared_dir}/run/docker/containerd:/run/docker/containerd" \
+		-v "${shared_dir}/run/nvidia:/run/nvidia" \
+		-v "${shared_dir}/usr/local/nvidia:/usr/local/nvidia" \
+		-v "${shared_dir}${CRIO_HOOKS_DIR}:${CRIO_HOOKS_DIR}" \
+		"${toolkit_container_image}" "-c" "$*"
+}
+
+

test/container/containerd_test.sh

@@ -0,0 +1,147 @@
+#! /bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readonly containerd_dind_ctr="container-config-containerd-dind-ctr-name"
+readonly containerd_test_ctr="container-config-containerd-test-ctr-name"
+readonly containerd_dind_socket="/run/nvidia/docker.sock"
+readonly containerd_dind_containerd_dir="/run/docker/containerd"
+
+testing::containerd::dind::setup() {
+	# Docker creates /etc/docker when starting
+	# by default there isn't any config in this directory (even after the daemon starts)
+	docker run -d --rm --privileged \
+		-v "${shared_dir}/etc/docker:/etc/docker" \
+		-v "${shared_dir}/run/nvidia:/run/nvidia" \
+		-v "${shared_dir}/usr/local/nvidia:/usr/local/nvidia" \
+		-v "${shared_dir}/run/docker/containerd:/run/docker/containerd" \
+		--name "${containerd_dind_ctr}" \
+		docker:stable-dind -H unix://${containerd_dind_socket}
+}
+
+testing::containerd::dind::exec() {
+	docker exec "${containerd_dind_ctr}" sh -c "$*"
+}
+
+testing::containerd::toolkit::run() {
+	local version=${1}
+
+	# We run ctr image list to ensure that containerd has successfully started in the docker-in-docker container
+	with_retry 5 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image list -q"
+
+	# Ensure that we can run some non GPU containers from within dind
+	with_retry 3 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image pull nvcr.io/nvidia/cuda:11.1-base; \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock run --rm --runtime=io.containerd.runtime.v1.linux nvcr.io/nvidia/cuda:11.1-base cuda echo foo"
+
+	# Share the volumes so that we can edit the config file and point to the new runtime
+	# Share the pid so that we can ask docker to reload its config
+	docker run --rm --privileged \
+		--volumes-from "${containerd_dind_ctr}" \
+		-v "${shared_dir}/etc/containerd/config_${version}.toml:${containerd_dind_containerd_dir}/containerd.toml" \
+		--pid "container:${containerd_dind_ctr}" \
+		-e "RUNTIME=containerd" \
+		-e "RUNTIME_ARGS=--config=${containerd_dind_containerd_dir}/containerd.toml --socket=${containerd_dind_containerd_dir}/containerd.sock" \
+		--name "${containerd_test_ctr}" \
+		"${toolkit_container_image}" "/usr/local/nvidia" "--no-daemon"
+
+	# We run ctr image list to ensure that containerd has successfully started in the docker-in-docker container
+	with_retry 5 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image list -q"
+
+	# Ensure that we haven't broken non GPU containers
+	with_retry 3 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image pull nvcr.io/nvidia/cuda:11.1-base; \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock run --rm --runtime=io.containerd.runtime.v1.linux nvcr.io/nvidia/cuda:11.1-base cuda echo foo"
+}
+
+# This test runs containerd setup and containerd cleanup in succession to ensure that the
+# config is restored correctly.
+testing::containerd::toolkit::test_config() {
+	local version=${1}
+
+	# We run ctr image list to ensure that containerd has successfully started in the docker-in-docker container
+	with_retry 5 5s testing::containerd::dind::exec " \
+		ctr --address=${containerd_dind_containerd_dir}/containerd.sock image list -q"
+
+	local input_config="${shared_dir}/etc/containerd/config_${version}.toml"
+	local output_config="${shared_dir}/output/config_${version}.toml"
+	local output_dir=$(dirname ${output_config})
+
+	mkdir -p ${output_dir}
+	cp -p "${input_config}" "${output_config}"
+
+	docker run --rm --privileged \
+		--volumes-from "${containerd_dind_ctr}" \
+		-v "${output_dir}:${output_dir}" \
+		--name "${containerd_test_ctr}" \
+		--entrypoint sh \
+		"${toolkit_container_image}" -c "containerd setup \
+			--config=${output_config} \
+			--socket=${containerd_dind_containerd_dir}/containerd.sock \
+			--restart-mode=NONE \
+				/usr/local/nvidia/toolkit"
+
+	# As a basic test we check that the config has changed
+	diff "${input_config}" "${output_config}" || test ${?} -ne 0
+	grep -q -E "^version = \d" "${output_config}"
+	grep -q -E "default_runtime_name = \"nvidia\"" "${output_config}"
+
+	docker run --rm --privileged \
+		--volumes-from "${containerd_dind_ctr}" \
+		-v "${output_dir}:${output_dir}" \
+		--name "${containerd_test_ctr}" \
+		--entrypoint sh \
+		"${toolkit_container_image}" -c "containerd cleanup \
+					--config=${output_config} \
+			--socket=${containerd_dind_containerd_dir}/containerd.sock \
+			--restart-mode=NONE \
+				/usr/local/nvidia/toolkit"
+
+	if [[ -s "${input_config}" ]]; then
+		# Compare the input and output config. These should be the same.
+		diff "${input_config}" "${output_config}" || true
+	else
+		# If the input config is empty, the output should not exist.
+		test ! -e "${output_config}"
+	fi
+}
+
+testing::containerd::main() {
+	testing::containerd::dind::setup
+
+	testing::containerd::toolkit::test_config empty
+	testing::containerd::toolkit::test_config v1
+	testing::containerd::toolkit::test_config v2
+
+	testing::containerd::cleanup
+
+	testing::containerd::dind::setup
+	testing::containerd::toolkit::run empty
+	testing::containerd::cleanup
+
+	testing::containerd::dind::setup
+	testing::containerd::toolkit::run v1
+	testing::containerd::cleanup
+
+	testing::containerd::dind::setup
+	testing::containerd::toolkit::run v2
+	testing::containerd::cleanup
+}
+
+testing::containerd::cleanup() {
+	docker kill "${containerd_dind_ctr}" &> /dev/null || true
+	docker kill "${containerd_test_ctr}" &> /dev/null || true
+}

test/container/crio_test.sh

@@ -0,0 +1,42 @@
+#! /bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+testing::crio::hook_created() {
+	testing::docker_run::toolkit::shell 'crio setup /run/nvidia/toolkit'
+
+	test ! -z "$(ls -A "${shared_dir}${CRIO_HOOKS_DIR}")"
+
+	cat "${shared_dir}${CRIO_HOOKS_DIR}/${CRIO_HOOK_FILENAME}" | \
+		jq -r '.hook.path' | grep -q "/run/nvidia/toolkit/"
+	test $? -eq 0
+	cat "${shared_dir}${CRIO_HOOKS_DIR}/${CRIO_HOOK_FILENAME}" | \
+		jq -r '.hook.env[0]' | grep -q ":/run/nvidia/toolkit"
+	test $? -eq 0
+}
+
+testing::crio::hook_cleanup() {
+	testing::docker_run::toolkit::shell 'crio cleanup'
+
+	test -z "$(ls -A "${shared_dir}${CRIO_HOOKS_DIR}")"
+}
+
+testing::crio::main() {
+	testing::crio::hook_created
+	testing::crio::hook_cleanup
+}
+
+testing::crio::cleanup() {
+	:
+}

test/container/docker_test.sh

@@ -0,0 +1,57 @@
+#! /bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+readonly docker_dind_ctr="container-config-docker-dind-ctr-name"
+readonly docker_test_ctr="container-config-docker-test-ctr-name"
+readonly docker_dind_socket="/run/nvidia/docker.sock"
+
+testing::docker::dind::setup() {
+	# Docker creates /etc/docker when starting
+	# by default there isn't any config in this directory (even after the daemon starts)
+	docker run -d --rm --privileged \
+		-v "${shared_dir}/etc/docker:/etc/docker" \
+		-v "${shared_dir}/run/nvidia:/run/nvidia" \
+		-v "${shared_dir}/usr/local/nvidia:/usr/local/nvidia" \
+		--name "${docker_dind_ctr}" \
+		docker:stable-dind -H unix://${docker_dind_socket}
+}
+
+testing::docker::dind::exec() {
+	docker exec "${docker_dind_ctr}" sh -c "$*"
+}
+
+testing::docker::toolkit::run() {
+	# Share the volumes so that we can edit the config file and point to the new runtime
+	# Share the pid so that we can ask docker to reload its config
+	docker run -d --rm --privileged \
+		--volumes-from "${docker_dind_ctr}" \
+		--pid "container:${docker_dind_ctr}" \
+		-e "RUNTIME_ARGS=--socket ${docker_dind_socket}" \
+		--name "${docker_test_ctr}" \
+		"${toolkit_container_image}" "/usr/local/nvidia" "--no-daemon"
+
+	# Ensure that we haven't broken non GPU containers
+	with_retry 3 5s testing::docker::dind::exec docker run -t alpine echo foo
+}
+
+testing::docker::main() {
+	testing::docker::dind::setup
+	testing::docker::toolkit::run
+}
+
+testing::docker::cleanup() {
+	docker kill "${docker_dind_ctr}" &> /dev/null || true
+	docker kill "${docker_test_ctr}" &> /dev/null || true
+}

test/container/main.sh

@@ -0,0 +1,77 @@
+#! /bin/bash
+# Copyright (c) 2019, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eEuo pipefail
+shopt -s lastpipe
+
+readonly basedir="$(dirname "$(realpath "$0")")"
+source "${basedir}/common.sh"
+
+source "${basedir}/toolkit_test.sh"
+source "${basedir}/docker_test.sh"
+source "${basedir}/crio_test.sh"
+source "${basedir}/containerd_test.sh"
+
+: ${CLEANUP:=true}
+
+usage() {
+	cat >&2 <<EOF
+Usage: $0 COMMAND [ARG...]
+
+Commands:
+  run SHARED_DIR TOOLKIT_CONTAINER_IMAGE [-c | --no-cleanup-on-error ]
+  clean SHARED_DIR
+EOF
+}
+
+if [ $# -lt 2 ]; then usage; exit 1; fi
+
+# We defined shared_dir here so that it can be used in cleanup
+readonly command=${1}; shift
+readonly shared_dir="${1}"; shift;
+
+case "${command}" in
+	clean) testing::cleanup; exit 0;;
+	run) ;;
+	*) usage; exit 0;;
+esac
+
+if [ $# -eq 0 ]; then usage; exit 1; fi
+
+readonly toolkit_container_image="${1}"; shift
+
+options=$(getopt -l no-cleanup-on-error -o c -- "$@")
+if [[ "$?" -ne 0 ]]; then usage; exit 1; fi
+
+# set options to positional parameters
+eval set -- "${options}"
+for opt in ${options}; do
+	case "${opt}" in
+	c | --no-cleanup-on-error) CLEANUP=false; shift;;
+	--) shift; break;;
+	esac
+done
+
+trap '"$CLEANUP" && testing::cleanup' ERR
+
+readonly test_cases="${TEST_CASES:-toolkit docker crio containerd}"
+
+testing::cleanup
+for tc in ${test_cases}; do
+	log INFO "=================Testing ${tc}================="
+	testing::setup
+	testing::${tc}::main "$@"
+	testing::cleanup
+done

test/container/shared/etc/containerd/config_v1.toml

@@ -0,0 +1,92 @@
+oom_score = 0
+root = "/var/lib/containerd"
+state = "/run/containerd"
+
+[cgroup]
+  path = ""
+
+[debug]
+  address = "/var/run/docker/containerd/containerd-debug.sock"
+  gid = 0
+  level = ""
+  uid = 0
+
+[grpc]
+  address = "/var/run/docker/containerd/containerd.sock"
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+  uid = 0
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[plugins]
+
+  [plugins.cgroups]
+    no_prometheus = false
+
+  [plugins.cri]
+    disable_proc_mount = false
+    enable_selinux = false
+    enable_tls_streaming = false
+    max_container_log_line_size = 16384
+    sandbox_image = "k8s.gcr.io/pause:3.1"
+    stats_collect_period = 10
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    systemd_cgroup = false
+
+    [plugins.cri.cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+
+    [plugins.cri.containerd]
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins.cri.containerd.default_runtime]
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = "io.containerd.runtime.v1.linux"
+
+      [plugins.cri.containerd.untrusted_workload_runtime]
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = ""
+
+    [plugins.cri.registry]
+
+      [plugins.cri.registry.mirrors]
+
+        [plugins.cri.registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+
+    [plugins.cri.x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+
+  [plugins.diff-service]
+    default = ["walking"]
+
+  [plugins.linux]
+    no_shim = false
+    runtime = "runc"
+    runtime_root = "/var/lib/docker/runc"
+    shim = "containerd-shim"
+    shim_debug = false
+
+  [plugins.opt]
+    path = "/opt/containerd"
+
+  [plugins.restart]
+    interval = "10s"
+
+  [plugins.scheduler]
+    deletion_threshold = 0
+    mutation_threshold = 100
+    pause_threshold = 0.02
+    schedule_delay = "0s"
+    startup_delay = "100ms"

test/container/shared/etc/containerd/config_v2.toml

@@ -0,0 +1,139 @@
+disabled_plugins = []
+oom_score = 0
+plugin_dir = ""
+required_plugins = []
+root = "/var/lib/containerd"
+state = "/run/containerd"
+version = 2
+
+[cgroup]
+  path = ""
+
+[debug]
+  address = "/var/run/docker/containerd/containerd-debug.sock"
+  gid = 0
+  level = ""
+  uid = 0
+
+[grpc]
+  address = "/var/run/docker/containerd/containerd.sock"
+  gid = 0
+  max_recv_message_size = 16777216
+  max_send_message_size = 16777216
+  tcp_address = ""
+  tcp_tls_cert = ""
+  tcp_tls_key = ""
+  uid = 0
+
+[metrics]
+  address = ""
+  grpc_histogram = false
+
+[plugins]
+
+  [plugins."io.containerd.gc.v1.scheduler"]
+    deletion_threshold = 0
+    mutation_threshold = 100
+    pause_threshold = 0.02
+    schedule_delay = "0s"
+    startup_delay = "100ms"
+
+  [plugins."io.containerd.grpc.v1.cri"]
+    disable_apparmor = false
+    disable_cgroup = false
+    disable_proc_mount = false
+    disable_tcp_service = true
+    enable_selinux = false
+    enable_tls_streaming = false
+    max_concurrent_downloads = 3
+    max_container_log_line_size = 16384
+    restrict_oom_score_adj = false
+    sandbox_image = "k8s.gcr.io/pause:3.1"
+    stats_collect_period = 10
+    stream_idle_timeout = "4h0m0s"
+    stream_server_address = "127.0.0.1"
+    stream_server_port = "0"
+    systemd_cgroup = false
+
+    [plugins."io.containerd.grpc.v1.cri".cni]
+      bin_dir = "/opt/cni/bin"
+      conf_dir = "/etc/cni/net.d"
+      conf_template = ""
+      max_conf_num = 1
+
+    [plugins."io.containerd.grpc.v1.cri".containerd]
+      default_runtime_name = "runc"
+      no_pivot = false
+      snapshotter = "overlayfs"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.default_runtime]
+        privileged_without_host_devices = false
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = ""
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.runtimes]
+
+        [plugins."io.containerd.grpc.v1.cri".containerd.runtimes.runc]
+          privileged_without_host_devices = false
+          runtime_engine = ""
+          runtime_root = ""
+          runtime_type = "io.containerd.runc.v1"
+
+      [plugins."io.containerd.grpc.v1.cri".containerd.untrusted_workload_runtime]
+        privileged_without_host_devices = false
+        runtime_engine = ""
+        runtime_root = ""
+        runtime_type = ""
+
+    [plugins."io.containerd.grpc.v1.cri".registry]
+
+      [plugins."io.containerd.grpc.v1.cri".registry.mirrors]
+
+        [plugins."io.containerd.grpc.v1.cri".registry.mirrors."docker.io"]
+          endpoint = ["https://registry-1.docker.io"]
+
+    [plugins."io.containerd.grpc.v1.cri".x509_key_pair_streaming]
+      tls_cert_file = ""
+      tls_key_file = ""
+
+  [plugins."io.containerd.internal.v1.opt"]
+    path = "/opt/containerd"
+
+  [plugins."io.containerd.internal.v1.restart"]
+    interval = "10s"
+
+  [plugins."io.containerd.metadata.v1.bolt"]
+    content_sharing_policy = "shared"
+
+  [plugins."io.containerd.monitor.v1.cgroups"]
+    no_prometheus = false
+
+  [plugins."io.containerd.runtime.v1.linux"]
+    no_shim = false
+    runtime = "runc"
+    runtime_root = "/var/lib/docker/runc"
+    shim = "containerd-shim"
+    shim_debug = false
+
+  [plugins."io.containerd.runtime.v2.task"]
+    platforms = ["linux/amd64"]
+
+  [plugins."io.containerd.service.v1.diff-service"]
+    default = ["walking"]
+
+  [plugins."io.containerd.snapshotter.v1.devmapper"]
+    base_image_size = ""
+    pool_name = ""
+    root_path = ""
+
+[timeouts]
+  "io.containerd.timeout.shim.cleanup" = "5s"
+  "io.containerd.timeout.shim.load" = "5s"
+  "io.containerd.timeout.shim.shutdown" = "3s"
+  "io.containerd.timeout.task.state" = "2s"
+
+[ttrpc]
+  address = ""
+  gid = 0
+  uid = 0

test/container/shared/etc/docker/daemon.json

@@ -0,0 +1,3 @@
+{
+  "registry-mirrors": ["https://mirror.gcr.io"]
+}

test/container/shared/run/nvidia/driver/usr/lib64/libnvidia-ml.so

@@ -0,0 +1 @@
+# This is a dummy lib file to test nvidia-runtime-experimental

test/container/toolkit_test.sh

@@ -0,0 +1,79 @@
+#! /bin/bash
+# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+testing::toolkit::install() {
+	local -r uid=$(id -u)
+	local -r gid=$(id -g)
+
+	local READLINK="readlink"
+	local -r platform=$(uname)
+	if [[ "${platform}" == "Darwin" ]]; then
+		READLINK="greadlink"
+	fi
+
+	testing::docker_run::toolkit::shell 'toolkit install /usr/local/nvidia/toolkit'
+	docker run --rm -v "${shared_dir}:/work" alpine sh -c "chown -R ${uid}:${gid} /work/"
+
+	# Ensure toolkit dir is correctly setup
+	test ! -z "$(ls -A "${shared_dir}/usr/local/nvidia/toolkit")"
+
+	test -L "${shared_dir}/usr/local/nvidia/toolkit/libnvidia-container.so.1"
+	test -e "$(${READLINK} -f "${shared_dir}/usr/local/nvidia/toolkit/libnvidia-container.so.1")"
+
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-cli"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-toolkit"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+
+	grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+	grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime"
+
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-cli.real"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-toolkit.real"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.real"
+
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime.experimental"
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
+
+	grep -q -E "nvidia driver modules are not yet loaded, invoking runc directly" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
+	grep -q -E "exec runc \".@\"" "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
+	grep -q -E "LD_LIBRARY_PATH=/run/nvidia/driver/usr/lib64:\\\$LD_LIBRARY_PATH " "${shared_dir}/usr/local/nvidia/toolkit/nvidia-container-runtime-experimental"
+
+	test -e "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
+
+	# Ensure that the config file has the required contents.
+	# NOTE: This assumes that RUN_DIR is '/run/nvidia'
+	local -r nvidia_run_dir="/run/nvidia"
+	grep -q -E "^\s*ldconfig = \"@${nvidia_run_dir}/driver/sbin/ldconfig(.real)?\"" "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
+	grep -q -E "^\s*root = \"${nvidia_run_dir}/driver\"" "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
+	grep -q -E "^\s*path = \"/usr/local/nvidia/toolkit/nvidia-container-cli\"" "${shared_dir}/usr/local/nvidia/toolkit/.config/nvidia-container-runtime/config.toml"
+}
+
+testing::toolkit::delete() {
+	testing::docker_run::toolkit::shell 'mkdir -p /usr/local/nvidia/delete-toolkit'
+	testing::docker_run::toolkit::shell 'touch /usr/local/nvidia/delete-toolkit/test.file'
+	testing::docker_run::toolkit::shell 'toolkit delete /usr/local/nvidia/delete-toolkit'
+
+	test ! -z "$(ls -A "${shared_dir}/usr/local/nvidia")"
+	test ! -e "${shared_dir}/usr/local/nvidia/delete-toolkit"
+}
+
+testing::toolkit::main() {
+	testing::toolkit::install
+	testing::toolkit::delete
+}
+
+testing::toolkit::cleanup() {
+	:
+}

test/input/config.clone3.json

@@ -0,0 +1,784 @@
+{
+  "ociVersion": "1.0.2-dev",
+  "process": {
+    "terminal": true,
+    "user": {
+      "uid": 0,
+      "gid": 0
+    },
+    "args": [
+      "sleep",
+      "60"
+    ],
+    "env": [
+      "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
+      "HOSTNAME=8de5efc6a95c",
+      "TERM=xterm"
+    ],
+    "cwd": "/",
+    "capabilities": {
+      "bounding": [
+        "CAP_CHOWN",
+        "CAP_DAC_OVERRIDE",
+        "CAP_FSETID",
+        "CAP_FOWNER",
+        "CAP_MKNOD",
+        "CAP_NET_RAW",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETFCAP",
+        "CAP_SETPCAP",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_SYS_CHROOT",
+        "CAP_KILL",
+        "CAP_AUDIT_WRITE"
+      ],
+      "effective": [
+        "CAP_CHOWN",
+        "CAP_DAC_OVERRIDE",
+        "CAP_FSETID",
+        "CAP_FOWNER",
+        "CAP_MKNOD",
+        "CAP_NET_RAW",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETFCAP",
+        "CAP_SETPCAP",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_SYS_CHROOT",
+        "CAP_KILL",
+        "CAP_AUDIT_WRITE"
+      ],
+      "inheritable": [
+        "CAP_CHOWN",
+        "CAP_DAC_OVERRIDE",
+        "CAP_FSETID",
+        "CAP_FOWNER",
+        "CAP_MKNOD",
+        "CAP_NET_RAW",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETFCAP",
+        "CAP_SETPCAP",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_SYS_CHROOT",
+        "CAP_KILL",
+        "CAP_AUDIT_WRITE"
+      ],
+      "permitted": [
+        "CAP_CHOWN",
+        "CAP_DAC_OVERRIDE",
+        "CAP_FSETID",
+        "CAP_FOWNER",
+        "CAP_MKNOD",
+        "CAP_NET_RAW",
+        "CAP_SETGID",
+        "CAP_SETUID",
+        "CAP_SETFCAP",
+        "CAP_SETPCAP",
+        "CAP_NET_BIND_SERVICE",
+        "CAP_SYS_CHROOT",
+        "CAP_KILL",
+        "CAP_AUDIT_WRITE"
+      ]
+    },
+    "apparmorProfile": "docker-default",
+    "oomScoreAdj": 0
+  },
+  "root": {
+    "path": "/var/lib/docker/overlay2/fbf92f54592ddb439159bc7eb25c865b9347a2d71d63b41b7b4e4a471847c84f/merged"
+  },
+  "hostname": "8de5efc6a95c",
+  "mounts": [
+    {
+      "destination": "/proc",
+      "type": "proc",
+      "source": "proc",
+      "options": [
+        "nosuid",
+        "noexec",
+        "nodev"
+      ]
+    },
+    {
+      "destination": "/dev",
+      "type": "tmpfs",
+      "source": "tmpfs",
+      "options": [
+        "nosuid",
+        "strictatime",
+        "mode=755",
+        "size=65536k"
+      ]
+    },
+    {
+      "destination": "/dev/pts",
+      "type": "devpts",
+      "source": "devpts",
+      "options": [
+        "nosuid",
+        "noexec",
+        "newinstance",
+        "ptmxmode=0666",
+        "mode=0620",
+        "gid=5"
+      ]
+    },
+    {
+      "destination": "/sys",
+      "type": "sysfs",
+      "source": "sysfs",
+      "options": [
+        "nosuid",
+        "noexec",
+        "nodev",
+        "ro"
+      ]
+    },
+    {
+      "destination": "/sys/fs/cgroup",
+      "type": "cgroup",
+      "source": "cgroup",
+      "options": [
+        "ro",
+        "nosuid",
+        "noexec",
+        "nodev"
+      ]
+    },
+    {
+      "destination": "/dev/mqueue",
+      "type": "mqueue",
+      "source": "mqueue",
+      "options": [
+        "nosuid",
+        "noexec",
+        "nodev"
+      ]
+    },
+    {
+      "destination": "/dev/shm",
+      "type": "tmpfs",
+      "source": "shm",
+      "options": [
+        "nosuid",
+        "noexec",
+        "nodev",
+        "mode=1777",
+        "size=67108864"
+      ]
+    },
+    {
+      "destination": "/etc/resolv.conf",
+      "type": "bind",
+      "source": "/var/lib/docker/containers/8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6/resolv.conf",
+      "options": [
+        "rbind",
+        "rprivate"
+      ]
+    },
+    {
+      "destination": "/etc/hostname",
+      "type": "bind",
+      "source": "/var/lib/docker/containers/8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6/hostname",
+      "options": [
+        "rbind",
+        "rprivate"
+      ]
+    },
+    {
+      "destination": "/etc/hosts",
+      "type": "bind",
+      "source": "/var/lib/docker/containers/8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6/hosts",
+      "options": [
+        "rbind",
+        "rprivate"
+      ]
+    }
+  ],
+  "hooks": {
+    "prestart": [
+      {
+        "path": "/proc/593/exe",
+        "args": [
+          "libnetwork-setkey",
+          "-exec-root=/var/run/docker",
+          "8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6",
+          "9967b9f7c4d4"
+        ]
+      }
+    ]
+  },
+  "linux": {
+    "sysctl": {
+      "net.ipv4.ip_unprivileged_port_start": "0"
+    },
+    "resources": {
+      "devices": [
+        {
+          "allow": false,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 1,
+          "minor": 5,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 1,
+          "minor": 3,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 1,
+          "minor": 9,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 1,
+          "minor": 8,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 5,
+          "minor": 0,
+          "access": "rwm"
+        },
+        {
+          "allow": true,
+          "type": "c",
+          "major": 5,
+          "minor": 1,
+          "access": "rwm"
+        },
+        {
+          "allow": false,
+          "type": "c",
+          "major": 10,
+          "minor": 229,
+          "access": "rwm"
+        }
+      ],
+      "memory": {
+        "disableOOMKiller": false
+      },
+      "cpu": {
+        "shares": 0
+      },
+      "blockIO": {
+        "weight": 0
+      }
+    },
+    "cgroupsPath": "/docker/8de5efc6a95c4ddae36dde7beb656ed5c7de912ecec2f628c42dbd0ef7bbeec6",
+    "namespaces": [
+      {
+        "type": "mount"
+      },
+      {
+        "type": "network"
+      },
+      {
+        "type": "uts"
+      },
+      {
+        "type": "pid"
+      },
+      {
+        "type": "ipc"
+      }
+    ],
+    "seccomp": {
+      "defaultAction": "SCMP_ACT_ERRNO",
+      "architectures": [
+        "SCMP_ARCH_X86_64",
+        "SCMP_ARCH_X86",
+        "SCMP_ARCH_X32"
+      ],
+      "syscalls": [
+        {
+          "names": [
+            "accept",
+            "accept4",
+            "access",
+            "adjtimex",
+            "alarm",
+            "bind",
+            "brk",
+            "capget",
+            "capset",
+            "chdir",
+            "chmod",
+            "chown",
+            "chown32",
+            "clock_adjtime",
+            "clock_adjtime64",
+            "clock_getres",
+            "clock_getres_time64",
+            "clock_gettime",
+            "clock_gettime64",
+            "clock_nanosleep",
+            "clock_nanosleep_time64",
+            "close",
+            "close_range",
+            "connect",
+            "copy_file_range",
+            "creat",
+            "dup",
+            "dup2",
+            "dup3",
+            "epoll_create",
+            "epoll_create1",
+            "epoll_ctl",
+            "epoll_ctl_old",
+            "epoll_pwait",
+            "epoll_pwait2",
+            "epoll_wait",
+            "epoll_wait_old",
+            "eventfd",
+            "eventfd2",
+            "execve",
+            "execveat",
+            "exit",
+            "exit_group",
+            "faccessat",
+            "faccessat2",
+            "fadvise64",
+            "fadvise64_64",
+            "fallocate",
+            "fanotify_mark",
+            "fchdir",
+            "fchmod",
+            "fchmodat",
+            "fchown",
+            "fchown32",
+            "fchownat",
+            "fcntl",
+            "fcntl64",
+            "fdatasync",
+            "fgetxattr",
+            "flistxattr",
+            "flock",
+            "fork",
+            "fremovexattr",
+            "fsetxattr",
+            "fstat",
+            "fstat64",
+            "fstatat64",
+            "fstatfs",
+            "fstatfs64",
+            "fsync",
+            "ftruncate",
+            "ftruncate64",
+            "futex",
+            "futex_time64",
+            "futimesat",
+            "getcpu",
+            "getcwd",
+            "getdents",
+            "getdents64",
+            "getegid",
+            "getegid32",
+            "geteuid",
+            "geteuid32",
+            "getgid",
+            "getgid32",
+            "getgroups",
+            "getgroups32",
+            "getitimer",
+            "getpeername",
+            "getpgid",
+            "getpgrp",
+            "getpid",
+            "getppid",
+            "getpriority",
+            "getrandom",
+            "getresgid",
+            "getresgid32",
+            "getresuid",
+            "getresuid32",
+            "getrlimit",
+            "get_robust_list",
+            "getrusage",
+            "getsid",
+            "getsockname",
+            "getsockopt",
+            "get_thread_area",
+            "gettid",
+            "gettimeofday",
+            "getuid",
+            "getuid32",
+            "getxattr",
+            "inotify_add_watch",
+            "inotify_init",
+            "inotify_init1",
+            "inotify_rm_watch",
+            "io_cancel",
+            "ioctl",
+            "io_destroy",
+            "io_getevents",
+            "io_pgetevents",
+            "io_pgetevents_time64",
+            "ioprio_get",
+            "ioprio_set",
+            "io_setup",
+            "io_submit",
+            "io_uring_enter",
+            "io_uring_register",
+            "io_uring_setup",
+            "ipc",
+            "kill",
+            "lchown",
+            "lchown32",
+            "lgetxattr",
+            "link",
+            "linkat",
+            "listen",
+            "listxattr",
+            "llistxattr",
+            "_llseek",
+            "lremovexattr",
+            "lseek",
+            "lsetxattr",
+            "lstat",
+            "lstat64",
+            "madvise",
+            "membarrier",
+            "memfd_create",
+            "mincore",
+            "mkdir",
+            "mkdirat",
+            "mknod",
+            "mknodat",
+            "mlock",
+            "mlock2",
+            "mlockall",
+            "mmap",
+            "mmap2",
+            "mprotect",
+            "mq_getsetattr",
+            "mq_notify",
+            "mq_open",
+            "mq_timedreceive",
+            "mq_timedreceive_time64",
+            "mq_timedsend",
+            "mq_timedsend_time64",
+            "mq_unlink",
+            "mremap",
+            "msgctl",
+            "msgget",
+            "msgrcv",
+            "msgsnd",
+            "msync",
+            "munlock",
+            "munlockall",
+            "munmap",
+            "nanosleep",
+            "newfstatat",
+            "_newselect",
+            "open",
+            "openat",
+            "openat2",
+            "pause",
+            "pidfd_open",
+            "pidfd_send_signal",
+            "pipe",
+            "pipe2",
+            "poll",
+            "ppoll",
+            "ppoll_time64",
+            "prctl",
+            "pread64",
+            "preadv",
+            "preadv2",
+            "prlimit64",
+            "pselect6",
+            "pselect6_time64",
+            "pwrite64",
+            "pwritev",
+            "pwritev2",
+            "read",
+            "readahead",
+            "readlink",
+            "readlinkat",
+            "readv",
+            "recv",
+            "recvfrom",
+            "recvmmsg",
+            "recvmmsg_time64",
+            "recvmsg",
+            "remap_file_pages",
+            "removexattr",
+            "rename",
+            "renameat",
+            "renameat2",
+            "restart_syscall",
+            "rmdir",
+            "rseq",
+            "rt_sigaction",
+            "rt_sigpending",
+            "rt_sigprocmask",
+            "rt_sigqueueinfo",
+            "rt_sigreturn",
+            "rt_sigsuspend",
+            "rt_sigtimedwait",
+            "rt_sigtimedwait_time64",
+            "rt_tgsigqueueinfo",
+            "sched_getaffinity",
+            "sched_getattr",
+            "sched_getparam",
+            "sched_get_priority_max",
+            "sched_get_priority_min",
+            "sched_getscheduler",
+            "sched_rr_get_interval",
+            "sched_rr_get_interval_time64",
+            "sched_setaffinity",
+            "sched_setattr",
+            "sched_setparam",
+            "sched_setscheduler",
+            "sched_yield",
+            "seccomp",
+            "select",
+            "semctl",
+            "semget",
+            "semop",
+            "semtimedop",
+            "semtimedop_time64",
+            "send",
+            "sendfile",
+            "sendfile64",
+            "sendmmsg",
+            "sendmsg",
+            "sendto",
+            "setfsgid",
+            "setfsgid32",
+            "setfsuid",
+            "setfsuid32",
+            "setgid",
+            "setgid32",
+            "setgroups",
+            "setgroups32",
+            "setitimer",
+            "setpgid",
+            "setpriority",
+            "setregid",
+            "setregid32",
+            "setresgid",
+            "setresgid32",
+            "setresuid",
+            "setresuid32",
+            "setreuid",
+            "setreuid32",
+            "setrlimit",
+            "set_robust_list",
+            "setsid",
+            "setsockopt",
+            "set_thread_area",
+            "set_tid_address",
+            "setuid",
+            "setuid32",
+            "setxattr",
+            "shmat",
+            "shmctl",
+            "shmdt",
+            "shmget",
+            "shutdown",
+            "sigaltstack",
+            "signalfd",
+            "signalfd4",
+            "sigprocmask",
+            "sigreturn",
+            "socket",
+            "socketcall",
+            "socketpair",
+            "splice",
+            "stat",
+            "stat64",
+            "statfs",
+            "statfs64",
+            "statx",
+            "symlink",
+            "symlinkat",
+            "sync",
+            "sync_file_range",
+            "syncfs",
+            "sysinfo",
+            "tee",
+            "tgkill",
+            "time",
+            "timer_create",
+            "timer_delete",
+            "timer_getoverrun",
+            "timer_gettime",
+            "timer_gettime64",
+            "timer_settime",
+            "timer_settime64",
+            "timerfd_create",
+            "timerfd_gettime",
+            "timerfd_gettime64",
+            "timerfd_settime",
+            "timerfd_settime64",
+            "times",
+            "tkill",
+            "truncate",
+            "truncate64",
+            "ugetrlimit",
+            "umask",
+            "uname",
+            "unlink",
+            "unlinkat",
+            "utime",
+            "utimensat",
+            "utimensat_time64",
+            "utimes",
+            "vfork",
+            "vmsplice",
+            "wait4",
+            "waitid",
+            "waitpid",
+            "write",
+            "writev"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        },
+        {
+          "names": [
+            "ptrace"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 0,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 8,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 131072,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 131080,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "personality"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 4294967295,
+              "op": "SCMP_CMP_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "arch_prctl"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        },
+        {
+          "names": [
+            "modify_ldt"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        },
+        {
+          "names": [
+            "clone"
+          ],
+          "action": "SCMP_ACT_ALLOW",
+          "args": [
+            {
+              "index": 0,
+              "value": 2114060288,
+              "op": "SCMP_CMP_MASKED_EQ"
+            }
+          ]
+        },
+        {
+          "names": [
+            "clone3"
+          ],
+          "action": "SCMP_ACT_ERRNO",
+          "errnoRet": 38
+        },
+        {
+          "names": [
+            "chroot"
+          ],
+          "action": "SCMP_ACT_ALLOW"
+        }
+      ]
+    },
+    "maskedPaths": [
+      "/proc/asound",
+      "/proc/acpi",
+      "/proc/kcore",
+      "/proc/keys",
+      "/proc/latency_stats",
+      "/proc/timer_list",
+      "/proc/timer_stats",
+      "/proc/sched_debug",
+      "/proc/scsi",
+      "/sys/firmware"
+    ],
+    "readonlyPaths": [
+      "/proc/bus",
+      "/proc/fs",
+      "/proc/irq",
+      "/proc/sys",
+      "/proc/sysrq-trigger"
+    ]
+  }
+}

tools/container/README.md

@@ -0,0 +1,77 @@
+## Introduction
+
+This repository contains tools that allow docker, containerd, or cri-o to be configured to use the NVIDIA Container Toolkit.
+
+*Note*: These were copied from the [`container-config` repository](https://gitlab.com/nvidia/container-toolkit/container-config/-/tree/383587f766a55177ede0e39e3810a974043e503e) are being migrated to commands installed with the NVIDIA Container Toolkit.
+
+These will be migrated into an upcoming `nvidia-ctk` CLI as required.
+
+### Docker
+
+After building the `docker` binary, run:
+```bash
+docker setup \
+    --runtime-name NAME \
+        /run/nvidia/toolkit
+```
+
+Configure the `nvidia-container-runtime` as a docker runtime named `NAME`. If the `--runtime-name` flag is not specified, this runtime would be called `nvidia`. A runtime named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
+
+Since `--set-as-default` is enabled by default, the specified runtime name will also be set as the default docker runtime. This can be disabled by explicityly specifying `--set-as-default=false`.
+
+**Note**: If `--runtime-name` is specified as `nvidia-experimental` explicitly, the `nvidia-experimental` runtime will be configured as the default runtime, with the `nvidia` runtime still configured and available for use.
+
+The following table describes the behaviour for different `--runtime-name` and `--set-as-default` flag combinations.
+
+| Flags                                                       | Installed Runtimes              | Default Runtime       |
+|-------------------------------------------------------------|:--------------------------------|:----------------------|
+| **NONE SPECIFIED**                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--runtime-name nvidia`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--runtime-name NAME`                                       | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--runtime-name nvidia-experimental`                        | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| `--set-as-default`                                          | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-name nvidia`                    | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-name NAME`                      | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--set-as-default --runtime-name nvidia-experimental`       | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+| `--set-as-default=false`                                    | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default=false --runtime-name NAME`                | `NAME`, `nvidia-experimental`   | **NOT SET**           |
+| `--set-as-default=false --runtime-name nvidia`              | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default=false --runtime-name nvidia-experimental` | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+
+These combinations also hold for the environment variables that map to the command line flags: `DOCKER_RUNTIME_NAME`, `DOCKER_SET_AS_DEFAULT`.
+
+### Containerd
+After running the `containerd` binary, run:
+```bash
+containerd setup \
+    --runtime-class NAME \
+        /run/nvidia/toolkit
+```
+
+Configure the `nvidia-container-runtime` as a runtime class named `NAME`. If the `--runtime-class` flag is not specified, this runtime would be called `nvidia`. A runtime class named `nvidia-experimental` will also be configured using the `nvidia-container-runtime-experimental` OCI-compliant runtime shim.
+
+Adding the `--set-as-default` flag as follows:
+```bash
+containerd setup \
+    --runtime-class NAME \
+    --set-as-default \
+        /run/nvidia/toolkit
+```
+will set the runtime class `NAME` (or `nvidia` if not specified) as the default runtime class.
+
+**Note**: If `--runtime-class` is specified as `nvidia-experimental` explicitly and `--set-as-default` is specified, the `nvidia-experimental` runtime will be configured as the default runtime class, with the `nvidia` runtime class still configured and available for use.
+
+The following table describes the behaviour for different `--runtime-class` and `--set-as-default` flag combinations.
+
+| Flags                                                  | Installed Runtime Classes       | Default Runtime Class |
+|--------------------------------------------------------|:--------------------------------|:----------------------|
+| **NONE SPECIFIED**                                     | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--runtime-class NAME`                                 | `NAME`, `nvidia-experimental`   | **NOT SET**           |
+| `--runtime-class nvidia`                               | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--runtime-class nvidia-experimental`                  | `nvidia`, `nvidia-experimental` | **NOT SET**           |
+| `--set-as-default`                                     | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-class NAME`                | `NAME`, `nvidia-experimental`   | `NAME`                |
+| `--set-as-default --runtime-class nvidia`              | `nvidia`, `nvidia-experimental` | `nvidia`              |
+| `--set-as-default --runtime-class nvidia-experimental` | `nvidia`, `nvidia-experimental` | `nvidia-experimental` |
+
+These combinations also hold for the environment variables that map to the command line flags.

tools/container/containerd/config.go

@@ -0,0 +1,116 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/pelletier/go-toml"
+)
+
+// UpdateReverter defines the interface for applying and reverting configurations
+type UpdateReverter interface {
+	Update(o *options) error
+	Revert(o *options) error
+}
+
+type config struct {
+	*toml.Tree
+	version   int64
+	cri       string
+	binaryKey string
+}
+
+// update adds the specified runtime class to the the containerd config.
+// if set-as default is specified, the runtime class is also set as the
+// default runtime.
+func (config *config) update(runtimeClass string, runtimeType string, runtimeBinary string, setAsDefault bool) {
+	config.Set("version", config.version)
+
+	runcPath := config.runcPath()
+	runtimeClassPath := config.runtimeClassPath(runtimeClass)
+
+	switch runc := config.GetPath(runcPath).(type) {
+	case *toml.Tree:
+		runc, _ = toml.Load(runc.String())
+		config.SetPath(runtimeClassPath, runc)
+	}
+
+	config.initRuntime(runtimeClassPath, runtimeType, runtimeBinary)
+
+	if setAsDefault {
+		defaultRuntimeNamePath := config.defaultRuntimeNamePath()
+		config.SetPath(defaultRuntimeNamePath, runtimeClass)
+	}
+}
+
+// revert removes the configuration applied in an update call.
+func (config *config) revert(runtimeClass string) {
+	runtimeClassPath := config.runtimeClassPath(runtimeClass)
+	defaultRuntimeNamePath := config.defaultRuntimeNamePath()
+
+	config.DeletePath(runtimeClassPath)
+	if runtime, ok := config.GetPath(defaultRuntimeNamePath).(string); ok {
+		if runtimeClass == runtime {
+			config.DeletePath(defaultRuntimeNamePath)
+		}
+	}
+
+	for i := 0; i < len(runtimeClassPath); i++ {
+		if runtimes, ok := config.GetPath(runtimeClassPath[:len(runtimeClassPath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(runtimeClassPath[:len(runtimeClassPath)-i])
+			}
+		}
+	}
+
+	if len(config.Keys()) == 1 && config.Keys()[0] == "version" {
+		config.Delete("version")
+	}
+}
+
+// initRuntime creates a runtime config if it does not exist and ensures that the
+// runtimes binary path is specified.
+func (config *config) initRuntime(path []string, runtimeType string, binary string) {
+	if config.GetPath(path) == nil {
+		config.SetPath(append(path, "runtime_type"), runtimeType)
+		config.SetPath(append(path, "runtime_root"), "")
+		config.SetPath(append(path, "runtime_engine"), "")
+		config.SetPath(append(path, "privileged_without_host_devices"), false)
+	}
+
+	binaryPath := append(path, "options", config.binaryKey)
+	config.SetPath(binaryPath, binary)
+}
+
+func (config config) runcPath() []string {
+	return config.runtimeClassPath("runc")
+}
+
+func (config config) runtimeClassBinaryPath(runtimeClass string) []string {
+	return append(config.runtimeClassPath(runtimeClass), "options", config.binaryKey)
+}
+
+func (config config) runtimeClassPath(runtimeClass string) []string {
+	return append(config.containerdPath(), "runtimes", runtimeClass)
+}
+
+func (config config) defaultRuntimeNamePath() []string {
+	return append(config.containerdPath(), "default_runtime_name")
+}
+
+func (config config) containerdPath() []string {
+	return []string{"plugins", config.cri, "containerd"}
+}

tools/container/containerd/config_v1.go

@@ -0,0 +1,126 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"path"
+
+	"github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+)
+
+// configV1 represents a V1 containerd config
+type configV1 struct {
+	config
+}
+
+func newConfigV1(cfg *toml.Tree) UpdateReverter {
+	c := configV1{
+		config: config{
+			Tree:      cfg,
+			version:   1,
+			cri:       "cri",
+			binaryKey: "Runtime",
+		},
+	}
+
+	return &c
+}
+
+// Update performs an update specific to v1 of the containerd config
+func (config *configV1) Update(o *options) error {
+
+	// For v1 config, the `default_runtime_name` setting is only supported
+	// for containerd version at least v1.3
+	supportsDefaultRuntimeName := !o.useLegacyConfig
+
+	defaultRuntime := o.getDefaultRuntime()
+
+	for runtimeClass, runtimeBinary := range o.getRuntimeBinaries() {
+		isDefaultRuntime := runtimeClass == defaultRuntime
+		config.update(runtimeClass, o.runtimeType, runtimeBinary, isDefaultRuntime && supportsDefaultRuntimeName)
+
+		if !isDefaultRuntime {
+			continue
+		}
+
+		if supportsDefaultRuntimeName {
+			defaultRuntimePath := append(config.containerdPath(), "default_runtime")
+			if config.GetPath(defaultRuntimePath) != nil {
+				log.Warnf("The setting of default_runtime (%v) in containerd is deprecated", defaultRuntimePath)
+			}
+			continue
+		}
+
+		log.Warnf("Setting default_runtime is deprecated")
+		defaultRuntimePath := append(config.containerdPath(), "default_runtime")
+		config.initRuntime(defaultRuntimePath, o.runtimeType, runtimeBinary)
+	}
+	return nil
+}
+
+// Revert performs a revert specific to v1 of the containerd config
+func (config *configV1) Revert(o *options) error {
+	defaultRuntimePath := append(config.containerdPath(), "default_runtime")
+	defaultRuntimeOptionsPath := append(defaultRuntimePath, "options")
+	if runtime, ok := config.GetPath(append(defaultRuntimeOptionsPath, "Runtime")).(string); ok {
+		for _, runtimeBinary := range o.getRuntimeBinaries() {
+			if path.Base(runtimeBinary) == path.Base(runtime) {
+				config.DeletePath(append(defaultRuntimeOptionsPath, "Runtime"))
+				break
+			}
+		}
+	}
+
+	if options, ok := config.GetPath(defaultRuntimeOptionsPath).(*toml.Tree); ok {
+		if len(options.Keys()) == 0 {
+			config.DeletePath(defaultRuntimeOptionsPath)
+		}
+	}
+
+	if runtime, ok := config.GetPath(defaultRuntimePath).(*toml.Tree); ok {
+		fields := []string{"runtime_type", "runtime_root", "runtime_engine", "privileged_without_host_devices"}
+		if len(runtime.Keys()) <= len(fields) {
+			matches := []string{}
+			for _, f := range fields {
+				e := runtime.Get(f)
+				if e != nil {
+					matches = append(matches, f)
+				}
+			}
+			if len(matches) == len(runtime.Keys()) {
+				for _, m := range matches {
+					runtime.Delete(m)
+				}
+			}
+		}
+	}
+
+	for i := 0; i < len(defaultRuntimePath); i++ {
+		if runtimes, ok := config.GetPath(defaultRuntimePath[:len(defaultRuntimePath)-i]).(*toml.Tree); ok {
+			if len(runtimes.Keys()) == 0 {
+				config.DeletePath(defaultRuntimePath[:len(defaultRuntimePath)-i])
+			}
+		}
+	}
+
+	for runtimeClass := range nvidiaRuntimeBinaries {
+		config.revert(runtimeClass)
+	}
+
+	return nil
+}

tools/container/containerd/config_v1_test.go

@@ -0,0 +1,365 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/pelletier/go-toml"
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateV1ConfigDefaultRuntime(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		legacyConfig                 bool
+		setAsDefault                 bool
+		runtimeClass                 string
+		expectedDefaultRuntimeName   interface{}
+		expectedDefaultRuntimeBinary interface{}
+	}{
+		{},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 false,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			runtimeClass:                 "NAME",
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime",
+		},
+		{
+			legacyConfig:                 true,
+			setAsDefault:                 true,
+			runtimeClass:                 "nvidia-experimental",
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: "/test/runtime/dir/nvidia-container-runtime-experimental",
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 false,
+			expectedDefaultRuntimeName:   nil,
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			expectedDefaultRuntimeName:   "nvidia",
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			runtimeClass:                 "NAME",
+			expectedDefaultRuntimeName:   "NAME",
+			expectedDefaultRuntimeBinary: nil,
+		},
+		{
+			legacyConfig:                 false,
+			setAsDefault:                 true,
+			runtimeClass:                 "nvidia-experimental",
+			expectedDefaultRuntimeName:   "nvidia-experimental",
+			expectedDefaultRuntimeBinary: nil,
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			useLegacyConfig: tc.legacyConfig,
+			setAsDefault:    tc.setAsDefault,
+			runtimeClass:    tc.runtimeClass,
+			runtimeType:     runtimeType,
+			runtimeDir:      runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(map[string]interface{}{})
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV1Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		defaultRuntimeName := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime_name"})
+		require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
+
+		defaultRuntime := config.GetPath([]string{"plugins", "cri", "containerd", "default_runtime"})
+		if tc.expectedDefaultRuntimeBinary == nil {
+			require.Nil(t, defaultRuntime, "%d: %v", i, tc)
+		} else {
+			expected, err := runtimeTomlConfigV1(tc.expectedDefaultRuntimeBinary.(string))
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(defaultRuntime.(*toml.Tree))
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, tc)
+		}
+
+	}
+}
+
+func TestUpdateV1Config(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+	const expectedVersion = int64(1)
+
+	expectedBinaries := []string{
+		"/test/runtime/dir/nvidia-container-runtime",
+		"/test/runtime/dir/nvidia-container-runtime-experimental",
+	}
+
+	testCases := []struct {
+		runtimeClass     string
+		expectedRuntimes []string
+	}{
+		{
+			runtimeClass:     "nvidia",
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "NAME",
+			expectedRuntimes: []string{"NAME", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "nvidia-experimental",
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: tc.runtimeClass,
+			runtimeType:  runtimeType,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(map[string]interface{}{})
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV1Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		version, ok := config.Get("version").(int64)
+		require.True(t, ok)
+		require.EqualValues(t, expectedVersion, version)
+
+		runtimes, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes"}).(*toml.Tree)
+		require.True(t, ok)
+
+		runtimeClasses := runtimes.Keys()
+		require.ElementsMatch(t, tc.expectedRuntimes, runtimeClasses, "%d: %v", i, tc)
+
+		for i, r := range tc.expectedRuntimes {
+			runtimeConfig := runtimes.Get(r)
+
+			expected, err := runtimeTomlConfigV1(expectedBinaries[i])
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(runtimeConfig)
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
+
+		}
+	}
+}
+
+func TestUpdateV1ConfigWithRuncPresent(t *testing.T) {
+	const runcBinary = "/runc-binary"
+	const runtimeDir = "/test/runtime/dir"
+	const expectedVersion = int64(1)
+
+	expectedBinaries := []string{
+		runcBinary,
+		"/test/runtime/dir/nvidia-container-runtime",
+		"/test/runtime/dir/nvidia-container-runtime-experimental",
+	}
+
+	testCases := []struct {
+		runtimeClass     string
+		expectedRuntimes []string
+	}{
+		{
+			runtimeClass:     "nvidia",
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "NAME",
+			expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "nvidia-experimental",
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: tc.runtimeClass,
+			runtimeType:  runtimeType,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(runcConfigMapV1("/runc-binary"))
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV1Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		version, ok := config.Get("version").(int64)
+		require.True(t, ok)
+		require.EqualValues(t, expectedVersion, version)
+
+		runtimes, ok := config.GetPath([]string{"plugins", "cri", "containerd", "runtimes"}).(*toml.Tree)
+		require.True(t, ok)
+
+		runtimeClasses := runtimes.Keys()
+		require.ElementsMatch(t, tc.expectedRuntimes, runtimeClasses, "%d: %v", i, tc)
+
+		for i, r := range tc.expectedRuntimes {
+			runtimeConfig := runtimes.Get(r)
+
+			expected, err := toml.TreeFromMap(runcRuntimeConfigMapV1(expectedBinaries[i]))
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(runtimeConfig)
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
+
+		}
+	}
+}
+
+func TestRevertV1Config(t *testing.T) {
+	testCases := []struct {
+		config map[string]interface {
+		}
+		expected map[string]interface{}
+	}{
+		{},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":              runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime-experimental"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(1),
+				"plugins": map[string]interface{}{
+					"cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":              runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-experimental": runtimeMapV1("/test/runtime/dir/nvidia-container-runtime-experimental"),
+							},
+							"default_runtime":      runtimeMapV1("/test/runtime/dir/nvidia-container-runtime"),
+							"default_runtime_name": "nvidia",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: "nvidia",
+		}
+
+		config, err := toml.TreeFromMap(tc.config)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		expected, err := toml.TreeFromMap(tc.expected)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = RevertV1Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		configContents, _ := toml.Marshal(config)
+		expectedContents, _ := toml.Marshal(expected)
+
+		require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
+	}
+}
+
+func runtimeTomlConfigV1(binary string) (*toml.Tree, error) {
+	return toml.TreeFromMap(runtimeMapV1(binary))
+}
+
+func runtimeMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"Runtime": binary,
+		},
+	}
+}
+
+func runcConfigMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"plugins": map[string]interface{}{
+			"cri": map[string]interface{}{
+				"containerd": map[string]interface{}{
+					"runtimes": map[string]interface{}{
+						"runc": runcRuntimeConfigMapV1(binary),
+					},
+				},
+			},
+		},
+	}
+}
+
+func runcRuntimeConfigMapV1(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    "runc_runtime_type",
+		"runtime_root":                    "runc_runtime_root",
+		"runtime_engine":                  "runc_runtime_engine",
+		"privileged_without_host_devices": true,
+		"options": map[string]interface{}{
+			"runc-option": "value",
+			"Runtime":     binary,
+		},
+	}
+}

tools/container/containerd/config_v2.go

@@ -0,0 +1,59 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"github.com/pelletier/go-toml"
+)
+
+// configV2 represents a V2 containerd config
+type configV2 struct {
+	config
+}
+
+func newConfigV2(cfg *toml.Tree) UpdateReverter {
+	c := configV2{
+		config: config{
+			Tree:      cfg,
+			version:   2,
+			cri:       "io.containerd.grpc.v1.cri",
+			binaryKey: "BinaryName",
+		},
+	}
+
+	return &c
+}
+
+// Update performs an update specific to v2 of the containerd config
+func (config *configV2) Update(o *options) error {
+	defaultRuntime := o.getDefaultRuntime()
+	for runtimeClass, runtimeBinary := range o.getRuntimeBinaries() {
+		setAsDefault := defaultRuntime == runtimeClass
+		config.update(runtimeClass, o.runtimeType, runtimeBinary, setAsDefault)
+	}
+
+	return nil
+}
+
+// Revert performs a revert specific to v2 of the containerd config
+func (config *configV2) Revert(o *options) error {
+	for runtimeClass := range o.getRuntimeBinaries() {
+		config.revert(runtimeClass)
+	}
+
+	return nil
+}

tools/container/containerd/config_v2_test.go

@@ -0,0 +1,329 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/pelletier/go-toml"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	runtimeType = "runtime_type"
+)
+
+func TestUpdateV2ConfigDefaultRuntime(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		setAsDefault               bool
+		runtimeClass               string
+		expectedDefaultRuntimeName interface{}
+	}{
+		{},
+		{
+			setAsDefault:               false,
+			runtimeClass:               "nvidia",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               false,
+			runtimeClass:               "NAME",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               false,
+			runtimeClass:               "nvidia-experimental",
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               true,
+			runtimeClass:               "nvidia",
+			expectedDefaultRuntimeName: "nvidia",
+		},
+		{
+			setAsDefault:               true,
+			runtimeClass:               "NAME",
+			expectedDefaultRuntimeName: "NAME",
+		},
+		{
+			setAsDefault:               true,
+			runtimeClass:               "nvidia-experimental",
+			expectedDefaultRuntimeName: "nvidia-experimental",
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			setAsDefault: tc.setAsDefault,
+			runtimeClass: tc.runtimeClass,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(map[string]interface{}{})
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV2Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		defaultRuntimeName := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "default_runtime_name"})
+		require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
+	}
+}
+
+func TestUpdateV2Config(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+	const expectedVersion = int64(2)
+
+	expectedBinaries := []string{
+		"/test/runtime/dir/nvidia-container-runtime",
+		"/test/runtime/dir/nvidia-container-runtime-experimental",
+	}
+
+	testCases := []struct {
+		runtimeClass     string
+		expectedRuntimes []string
+	}{
+		{
+			runtimeClass:     "nvidia",
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "NAME",
+			expectedRuntimes: []string{"NAME", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "nvidia-experimental",
+			expectedRuntimes: []string{"nvidia", "nvidia-experimental"},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: tc.runtimeClass,
+			runtimeType:  runtimeType,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(map[string]interface{}{})
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV2Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		version, ok := config.Get("version").(int64)
+		require.True(t, ok)
+		require.EqualValues(t, expectedVersion, version, "%d: %v", i, tc)
+
+		runtimes, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes"}).(*toml.Tree)
+		require.True(t, ok)
+
+		runtimeClasses := runtimes.Keys()
+		require.ElementsMatch(t, tc.expectedRuntimes, runtimeClasses, "%d: %v", i, tc)
+
+		for i, r := range tc.expectedRuntimes {
+			runtimeConfig := runtimes.Get(r)
+
+			expected, err := runtimeTomlConfigV2(expectedBinaries[i])
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(runtimeConfig)
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
+
+		}
+	}
+
+}
+
+func TestUpdateV2ConfigWithRuncPresent(t *testing.T) {
+	const runcBinary = "/runc-binary"
+	const runtimeDir = "/test/runtime/dir"
+	const expectedVersion = int64(2)
+
+	expectedBinaries := []string{
+		runcBinary,
+		"/test/runtime/dir/nvidia-container-runtime",
+		"/test/runtime/dir/nvidia-container-runtime-experimental",
+	}
+
+	testCases := []struct {
+		runtimeClass     string
+		expectedRuntimes []string
+	}{
+		{
+			runtimeClass:     "nvidia",
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "NAME",
+			expectedRuntimes: []string{"runc", "NAME", "nvidia-experimental"},
+		},
+		{
+			runtimeClass:     "nvidia-experimental",
+			expectedRuntimes: []string{"runc", "nvidia", "nvidia-experimental"},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: tc.runtimeClass,
+			runtimeType:  runtimeType,
+			runtimeDir:   runtimeDir,
+		}
+
+		config, err := toml.TreeFromMap(runcConfigMapV2("/runc-binary"))
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = UpdateV2Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		version, ok := config.Get("version").(int64)
+		require.True(t, ok)
+		require.EqualValues(t, expectedVersion, version)
+
+		runtimes, ok := config.GetPath([]string{"plugins", "io.containerd.grpc.v1.cri", "containerd", "runtimes"}).(*toml.Tree)
+		require.True(t, ok, "%d: %v", i, tc)
+
+		runtimeClasses := runtimes.Keys()
+		require.ElementsMatch(t, tc.expectedRuntimes, runtimeClasses, "%d: %v", i, tc)
+
+		for i, r := range tc.expectedRuntimes {
+			runtimeConfig := runtimes.Get(r)
+
+			expected, err := toml.TreeFromMap(runcRuntimeConfigMapV2(expectedBinaries[i]))
+			require.NoError(t, err, "%d: %v", i, tc)
+
+			configContents, _ := toml.Marshal(runtimeConfig)
+			expectedContents, _ := toml.Marshal(expected)
+
+			require.Equal(t, string(expectedContents), string(configContents), "%d: %v: %v", i, r, tc)
+
+		}
+	}
+}
+
+func TestRevertV2Config(t *testing.T) {
+	testCases := []struct {
+		config map[string]interface {
+		}
+		expected map[string]interface{}
+	}{
+		{},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":              runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime-experimental"),
+							},
+						},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"version": int64(2),
+				"plugins": map[string]interface{}{
+					"io.containerd.grpc.v1.cri": map[string]interface{}{
+						"containerd": map[string]interface{}{
+							"runtimes": map[string]interface{}{
+								"nvidia":              runtimeMapV2("/test/runtime/dir/nvidia-container-runtime"),
+								"nvidia-experimental": runtimeMapV2("/test/runtime/dir/nvidia-container-runtime-experimental"),
+							},
+							"default_runtime_name": "nvidia",
+						},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			runtimeClass: "nvidia",
+		}
+
+		config, err := toml.TreeFromMap(tc.config)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		expected, err := toml.TreeFromMap(tc.expected)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		err = RevertV2Config(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		configContents, _ := toml.Marshal(config)
+		expectedContents, _ := toml.Marshal(expected)
+
+		require.Equal(t, string(expectedContents), string(configContents), "%d: %v", i, tc)
+	}
+}
+
+func runtimeTomlConfigV2(binary string) (*toml.Tree, error) {
+	return toml.TreeFromMap(runtimeMapV2(binary))
+}
+
+func runtimeMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    runtimeType,
+		"runtime_root":                    "",
+		"runtime_engine":                  "",
+		"privileged_without_host_devices": false,
+		"options": map[string]interface{}{
+			"BinaryName": binary,
+		},
+	}
+}
+
+func runcConfigMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"plugins": map[string]interface{}{
+			"io.containerd.grpc.v1.cri": map[string]interface{}{
+				"containerd": map[string]interface{}{
+					"runtimes": map[string]interface{}{
+						"runc": runcRuntimeConfigMapV2(binary),
+					},
+				},
+			},
+		},
+	}
+}
+
+func runcRuntimeConfigMapV2(binary string) map[string]interface{} {
+	return map[string]interface{}{
+		"runtime_type":                    "runc_runtime_type",
+		"runtime_root":                    "runc_runtime_root",
+		"runtime_engine":                  "runc_runtime_engine",
+		"privileged_without_host_devices": true,
+		"options": map[string]interface{}{
+			"runc-option": "value",
+			"BinaryName":  binary,
+		},
+	}
+}

tools/container/containerd/containerd.go

@@ -0,0 +1,587 @@
+/**
+# Copyright (c) 2020-2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/containerd/containerd/plugin"
+	toml "github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+)
+
+const (
+	restartModeSignal  = "signal"
+	restartModeSystemd = "systemd"
+	restartModeNone    = "NONE"
+
+	nvidiaRuntimeName               = "nvidia"
+	nvidiaRuntimeBinary             = "nvidia-container-runtime"
+	nvidiaExperimentalRuntimeName   = "nvidia-experimental"
+	nvidiaExperimentalRuntimeBinary = "nvidia-container-runtime-experimental"
+
+	defaultConfig        = "/etc/containerd/config.toml"
+	defaultSocket        = "/run/containerd/containerd.sock"
+	defaultRuntimeClass  = "nvidia"
+	defaultRuntmeType    = plugin.RuntimeRuncV2
+	defaultSetAsDefault  = true
+	defaultRestartMode   = restartModeSignal
+	defaultHostRootMount = "/host"
+
+	reloadBackoff     = 5 * time.Second
+	maxReloadAttempts = 6
+
+	socketMessageToGetPID = ""
+)
+
+// nvidiaRuntimeBinaries defines a map of runtime names to binary names
+var nvidiaRuntimeBinaries = map[string]string{
+	nvidiaRuntimeName:             nvidiaRuntimeBinary,
+	nvidiaExperimentalRuntimeName: nvidiaExperimentalRuntimeBinary,
+}
+
+// options stores the configuration from the command line or environment variables
+type options struct {
+	config          string
+	socket          string
+	runtimeClass    string
+	runtimeType     string
+	setAsDefault    bool
+	restartMode     string
+	hostRootMount   string
+	runtimeDir      string
+	useLegacyConfig bool
+}
+
+func main() {
+	options := options{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "containerd"
+	c.Usage = "Update a containerd config with the nvidia-container-runtime"
+	c.Version = "0.1.0"
+
+	// Create the 'setup' subcommand
+	setup := cli.Command{}
+	setup.Name = "setup"
+	setup.Usage = "Trigger a containerd config to be updated"
+	setup.ArgsUsage = "<runtime_dirname>"
+	setup.Action = func(c *cli.Context) error {
+		return Setup(c, &options)
+	}
+
+	// Create the 'cleanup' subcommand
+	cleanup := cli.Command{}
+	cleanup.Name = "cleanup"
+	cleanup.Usage = "Trigger any updates made to a containerd config to be undone"
+	cleanup.ArgsUsage = "<runtime_dirname>"
+	cleanup.Action = func(c *cli.Context) error {
+		return Cleanup(c, &options)
+	}
+
+	// Register the subcommands with the top-level CLI
+	c.Commands = []*cli.Command{
+		&setup,
+		&cleanup,
+	}
+
+	// Setup common flags across both subcommands. All subcommands get the same
+	// set of flags even if they don't use some of them. This is so that we
+	// only require the user to specify one set of flags for both 'startup'
+	// and 'cleanup' to simplify things.
+	commonFlags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config",
+			Aliases:     []string{"c"},
+			Usage:       "Path to the containerd config file",
+			Value:       defaultConfig,
+			Destination: &options.config,
+			EnvVars:     []string{"CONTAINERD_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "socket",
+			Aliases:     []string{"s"},
+			Usage:       "Path to the containerd socket file",
+			Value:       defaultSocket,
+			Destination: &options.socket,
+			EnvVars:     []string{"CONTAINERD_SOCKET"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-class",
+			Aliases:     []string{"r"},
+			Usage:       "The name of the runtime class to set for the nvidia-container-runtime",
+			Value:       defaultRuntimeClass,
+			Destination: &options.runtimeClass,
+			EnvVars:     []string{"CONTAINERD_RUNTIME_CLASS"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-type",
+			Usage:       "The runtime_type to use for the configured runtime classes",
+			Value:       defaultRuntmeType,
+			Destination: &options.runtimeType,
+			EnvVars:     []string{"CONTAINERD_RUNTIME_TYPE"},
+		},
+		// The flags below are only used by the 'setup' command.
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Aliases:     []string{"d"},
+			Usage:       "Set nvidia-container-runtime as the default runtime",
+			Value:       defaultSetAsDefault,
+			Destination: &options.setAsDefault,
+			EnvVars:     []string{"CONTAINERD_SET_AS_DEFAULT"},
+			Hidden:      true,
+		},
+		&cli.StringFlag{
+			Name:        "restart-mode",
+			Usage:       "Specify how containerd should be restarted; [signal | systemd]",
+			Value:       defaultRestartMode,
+			Destination: &options.restartMode,
+			EnvVars:     []string{"CONTAINERD_RESTART_MODE"},
+		},
+		&cli.StringFlag{
+			Name:        "host-root",
+			Usage:       "Specify the path to the host root to be used when restarting containerd using systemd",
+			Value:       defaultHostRootMount,
+			Destination: &options.hostRootMount,
+			EnvVars:     []string{"HOST_ROOT_MOUNT"},
+		},
+		&cli.BoolFlag{
+			Name:        "use-legacy-config",
+			Usage:       "Specify whether a legacy (pre v1.3) config should be used",
+			Destination: &options.useLegacyConfig,
+			EnvVars:     []string{"CONTAINERD_USE_LEGACY_CONFIG"},
+		},
+	}
+
+	// Update the subcommand flags with the common subcommand flags
+	setup.Flags = append([]cli.Flag{}, commonFlags...)
+	cleanup.Flags = append([]cli.Flag{}, commonFlags...)
+
+	// Run the top-level CLI
+	if err := c.Run(os.Args); err != nil {
+		log.Fatal(fmt.Errorf("Error: %v", err))
+	}
+}
+
+// Setup updates a containerd configuration to include the nvidia-containerd-runtime and reloads it
+func Setup(c *cli.Context, o *options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	runtimeDir, err := ParseArgs(c)
+	if err != nil {
+		return fmt.Errorf("unable to parse args: %v", err)
+	}
+	o.runtimeDir = runtimeDir
+
+	cfg, err := LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	version, err := ParseVersion(cfg, o.useLegacyConfig)
+	if err != nil {
+		return fmt.Errorf("unable to parse version: %v", err)
+	}
+
+	err = UpdateConfig(cfg, o, version)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = FlushConfig(o.config, cfg)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = RestartContainerd(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart containerd: %v", err)
+	}
+
+	log.Infof("Completed 'setup' for %v", c.App.Name)
+
+	return nil
+}
+
+// Cleanup reverts a containerd configuration to remove the nvidia-containerd-runtime and reloads it
+func Cleanup(c *cli.Context, o *options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	_, err := ParseArgs(c)
+	if err != nil {
+		return fmt.Errorf("unable to parse args: %v", err)
+	}
+
+	cfg, err := LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	version, err := ParseVersion(cfg, o.useLegacyConfig)
+	if err != nil {
+		return fmt.Errorf("unable to parse version: %v", err)
+	}
+
+	err = RevertConfig(cfg, o, version)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = FlushConfig(o.config, cfg)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = RestartContainerd(o)
+	if err != nil {
+		return fmt.Errorf("unable to restart containerd: %v", err)
+	}
+
+	log.Infof("Completed 'cleanup' for %v", c.App.Name)
+
+	return nil
+}
+
+// ParseArgs parses the command line arguments to the CLI
+func ParseArgs(c *cli.Context) (string, error) {
+	args := c.Args()
+
+	log.Infof("Parsing arguments: %v", args.Slice())
+	if args.Len() != 1 {
+		return "", fmt.Errorf("incorrect number of arguments")
+	}
+	runtimeDir := args.Get(0)
+	log.Infof("Successfully parsed arguments")
+
+	return runtimeDir, nil
+}
+
+// LoadConfig loads the containerd config from disk
+func LoadConfig(config string) (*toml.Tree, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	configFile := config
+	if os.IsNotExist(err) {
+		configFile = "/dev/null"
+		log.Infof("Config file does not exist, creating new one")
+	}
+
+	cfg, err := toml.LoadFile(configFile)
+	if err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+
+	return cfg, nil
+}
+
+// ParseVersion parses the version field out of the containerd config
+func ParseVersion(config *toml.Tree, useLegacyConfig bool) (int, error) {
+	var defaultVersion int
+	if !useLegacyConfig {
+		defaultVersion = 2
+	} else {
+		defaultVersion = 1
+	}
+
+	var version int
+	switch v := config.Get("version").(type) {
+	case nil:
+		switch len(config.Keys()) {
+		case 0: // No config exists, or the config file is empty, use version inferred from containerd
+			version = defaultVersion
+		default: // A config file exists, has content, and no version is set
+			version = 1
+		}
+	case int64:
+		version = int(v)
+	default:
+		return -1, fmt.Errorf("unsupported type for version field: %v", v)
+	}
+	log.Infof("Config version: %v", version)
+
+	if version == 1 {
+		log.Warnf("Support for containerd config version 1 is deprecated")
+	}
+
+	return version, nil
+}
+
+// UpdateConfig updates the containerd config to include the nvidia-container-runtime
+func UpdateConfig(config *toml.Tree, o *options, version int) error {
+	var err error
+
+	log.Infof("Updating config")
+	switch version {
+	case 1:
+		err = UpdateV1Config(config, o)
+	case 2:
+		err = UpdateV2Config(config, o)
+	default:
+		err = fmt.Errorf("unsupported containerd config version: %v", version)
+	}
+	if err != nil {
+		return err
+	}
+	log.Infof("Successfully updated config")
+
+	return nil
+}
+
+// RevertConfig reverts the containerd config to remove the nvidia-container-runtime
+func RevertConfig(config *toml.Tree, o *options, version int) error {
+	var err error
+
+	log.Infof("Reverting config")
+	switch version {
+	case 1:
+		err = RevertV1Config(config, o)
+	case 2:
+		err = RevertV2Config(config, o)
+	default:
+		err = fmt.Errorf("unsupported containerd config version: %v", version)
+	}
+	if err != nil {
+		return err
+	}
+	log.Infof("Successfully reverted config")
+
+	return nil
+}
+
+// UpdateV1Config performs an update specific to v1 of the containerd config
+func UpdateV1Config(config *toml.Tree, o *options) error {
+	c := newConfigV1(config)
+	return c.Update(o)
+}
+
+// RevertV1Config performs a revert specific to v1 of the containerd config
+func RevertV1Config(config *toml.Tree, o *options) error {
+	c := newConfigV1(config)
+	return c.Revert(o)
+}
+
+// UpdateV2Config performs an update specific to v2 of the containerd config
+func UpdateV2Config(config *toml.Tree, o *options) error {
+	c := newConfigV2(config)
+	return c.Update(o)
+}
+
+// RevertV2Config performs a revert specific to v2 of the containerd config
+func RevertV2Config(config *toml.Tree, o *options) error {
+	c := newConfigV2(config)
+	return c.Revert(o)
+}
+
+// FlushConfig flushes the updated/reverted config out to disk
+func FlushConfig(config string, cfg *toml.Tree) error {
+	log.Infof("Flushing config")
+
+	output, err := cfg.ToTomlString()
+	if err != nil {
+		return fmt.Errorf("unable to convert to TOML: %v", err)
+	}
+
+	switch len(output) {
+	case 0:
+		err := os.Remove(config)
+		if err != nil {
+			return fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		log.Infof("Config empty, removing file")
+	default:
+		f, err := os.Create(config)
+		if err != nil {
+			return fmt.Errorf("unable to open '%v' for writing: %v", config, err)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(output)
+		if err != nil {
+			return fmt.Errorf("unable to write output: %v", err)
+		}
+	}
+
+	log.Infof("Successfully flushed config")
+
+	return nil
+}
+
+// RestartContainerd restarts containerd depending on the value of restartModeFlag
+func RestartContainerd(o *options) error {
+	switch o.restartMode {
+	case restartModeNone:
+		log.Warnf("Skipping sending signal to containerd due to --restart-mode=%v", o.restartMode)
+		return nil
+	case restartModeSignal:
+		err := SignalContainerd(o)
+		if err != nil {
+			return fmt.Errorf("unable to signal containerd: %v", err)
+		}
+	case restartModeSystemd:
+		return RestartContainerdSystemd(o.hostRootMount)
+	default:
+		return fmt.Errorf("Invalid restart mode specified: %v", o.restartMode)
+	}
+
+	return nil
+}
+
+// SignalContainerd sends a SIGHUP signal to the containerd daemon
+func SignalContainerd(o *options) error {
+	log.Infof("Sending SIGHUP signal to containerd")
+
+	// Wrap the logic to perform the SIGHUP in a function so we can retry it on failure
+	retriable := func() error {
+		conn, err := net.Dial("unix", o.socket)
+		if err != nil {
+			return fmt.Errorf("unable to dial: %v", err)
+		}
+		defer conn.Close()
+
+		sconn, err := conn.(*net.UnixConn).SyscallConn()
+		if err != nil {
+			return fmt.Errorf("unable to get syscall connection: %v", err)
+		}
+
+		err1 := sconn.Control(func(fd uintptr) {
+			err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
+		})
+		if err1 != nil {
+			return fmt.Errorf("unable to issue call on socket fd: %v", err1)
+		}
+		if err != nil {
+			return fmt.Errorf("unable to SetsockoptInt on socket fd: %v", err)
+		}
+
+		_, _, err = conn.(*net.UnixConn).WriteMsgUnix([]byte(socketMessageToGetPID), nil, nil)
+		if err != nil {
+			return fmt.Errorf("unable to WriteMsgUnix on socket fd: %v", err)
+		}
+
+		oob := make([]byte, 1024)
+		_, oobn, _, _, err := conn.(*net.UnixConn).ReadMsgUnix(nil, oob)
+		if err != nil {
+			return fmt.Errorf("unable to ReadMsgUnix on socket fd: %v", err)
+		}
+
+		oob = oob[:oobn]
+		scm, err := syscall.ParseSocketControlMessage(oob)
+		if err != nil {
+			return fmt.Errorf("unable to ParseSocketControlMessage from message received on socket fd: %v", err)
+		}
+
+		ucred, err := syscall.ParseUnixCredentials(&scm[0])
+		if err != nil {
+			return fmt.Errorf("unable to ParseUnixCredentials from message received on socket fd: %v", err)
+		}
+
+		err = syscall.Kill(int(ucred.Pid), syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("unable to send SIGHUP to 'containerd' process: %v", err)
+		}
+
+		return nil
+	}
+
+	// Try to send a SIGHUP up to maxReloadAttempts times
+	var err error
+	for i := 0; i < maxReloadAttempts; i++ {
+		err = retriable()
+		if err == nil {
+			break
+		}
+		if i == maxReloadAttempts-1 {
+			break
+		}
+		log.Warnf("Error signaling containerd, attempt %v/%v: %v", i+1, maxReloadAttempts, err)
+		time.Sleep(reloadBackoff)
+	}
+	if err != nil {
+		log.Warnf("Max retries reached %v/%v, aborting", maxReloadAttempts, maxReloadAttempts)
+		return err
+	}
+
+	log.Infof("Successfully signaled containerd")
+
+	return nil
+}
+
+// RestartContainerdSystemd restarts containerd using systemctl
+func RestartContainerdSystemd(hostRootMount string) error {
+	log.Infof("Restarting containerd using systemd and host root mounted at %v", hostRootMount)
+
+	command := "chroot"
+	args := []string{hostRootMount, "systemctl", "restart", "containerd"}
+
+	cmd := exec.Command(command, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error restarting containerd using systemd: %v", err)
+	}
+
+	return nil
+}
+
+// getDefaultRuntime returns the default runtime for the configured options.
+// If the configuration is invalid or the default runtimes should not be set
+// the empty string is returned.
+func (o options) getDefaultRuntime() string {
+	if o.setAsDefault {
+		if o.runtimeClass == nvidiaExperimentalRuntimeName {
+			return nvidiaExperimentalRuntimeName
+		}
+		if o.runtimeClass == "" {
+			return defaultRuntimeClass
+		}
+		return o.runtimeClass
+	}
+	return ""
+}
+
+// getRuntimeBinaries returns a map of runtime names to binary paths. This includes the
+// renaming of the `nvidia` runtime as per the --runtime-class command line flag.
+func (o options) getRuntimeBinaries() map[string]string {
+	runtimeBinaries := make(map[string]string)
+
+	for rt, bin := range nvidiaRuntimeBinaries {
+		runtime := rt
+		if o.runtimeClass != "" && o.runtimeClass != nvidiaExperimentalRuntimeName && runtime == defaultRuntimeClass {
+			runtime = o.runtimeClass
+		}
+
+		runtimeBinaries[runtime] = filepath.Join(o.runtimeDir, bin)
+	}
+
+	return runtimeBinaries
+}

tools/container/containerd/containerd_test.go

@@ -0,0 +1,106 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestOptions(t *testing.T) {
+	testCases := []struct {
+		options                 options
+		expectedDefaultRuntime  string
+		expectedRuntimeBinaries map[string]string
+	}{
+		{
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: true,
+			},
+			expectedDefaultRuntime: "nvidia",
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: true,
+				runtimeClass: "nvidia",
+			},
+			expectedDefaultRuntime: "nvidia",
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: true,
+				runtimeClass: "NAME",
+			},
+			expectedDefaultRuntime: "NAME",
+			expectedRuntimeBinaries: map[string]string{
+				"NAME":                "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: false,
+				runtimeClass: "NAME",
+			},
+			expectedRuntimeBinaries: map[string]string{
+				"NAME":                "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: true,
+				runtimeClass: "nvidia-experimental",
+			},
+			expectedDefaultRuntime: "nvidia-experimental",
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+		{
+			options: options{
+				setAsDefault: false,
+				runtimeClass: "nvidia-experimental",
+			},
+			expectedRuntimeBinaries: map[string]string{
+				"nvidia":              "nvidia-container-runtime",
+				"nvidia-experimental": "nvidia-container-runtime-experimental",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		require.Equal(t, tc.expectedDefaultRuntime, tc.options.getDefaultRuntime(), "%d: %v", i, tc)
+		require.EqualValues(t, tc.expectedRuntimeBinaries, tc.options.getRuntimeBinaries(), "%d: %v", i, tc)
+	}
+}

tools/container/crio/crio.go

@@ -0,0 +1,185 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	hooks "github.com/containers/podman/v2/pkg/hooks/1.0.0"
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+)
+
+const (
+	defaultHooksDir     = "/usr/share/containers/oci/hooks.d"
+	defaultHookFilename = "oci-nvidia-hook.json"
+)
+
+var hooksDirFlag string
+var hookFilenameFlag string
+var tooklitDirArg string
+
+func main() {
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "crio"
+	c.Usage = "Update cri-o hooks to include the NVIDIA runtime hook"
+	c.ArgsUsage = "<toolkit_dirname>"
+	c.Version = "0.1.0"
+
+	// Create the 'setup' subcommand
+	setup := cli.Command{}
+	setup.Name = "setup"
+	setup.Usage = "Create the cri-o hook required to run NVIDIA GPU containers"
+	setup.ArgsUsage = "<toolkit_dirname>"
+	setup.Action = Setup
+	setup.Before = ParseArgs
+
+	// Create the 'cleanup' subcommand
+	cleanup := cli.Command{}
+	cleanup.Name = "cleanup"
+	cleanup.Usage = "Remove the NVIDIA cri-o hook"
+	cleanup.Action = Cleanup
+
+	// Register the subcommands with the top-level CLI
+	c.Commands = []*cli.Command{
+		&setup,
+		&cleanup,
+	}
+
+	// Setup common flags across both subcommands. All subcommands get the same
+	// set of flags even if they don't use some of them. This is so that we
+	// only require the user to specify one set of flags for both 'startup'
+	// and 'cleanup' to simplify things.
+	commonFlags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "hooks-dir",
+			Aliases:     []string{"d"},
+			Usage:       "path to the cri-o hooks directory",
+			Value:       defaultHooksDir,
+			Destination: &hooksDirFlag,
+			EnvVars:     []string{"CRIO_HOOKS_DIR"},
+			DefaultText: defaultHooksDir,
+		},
+		&cli.StringFlag{
+			Name:        "hook-filename",
+			Aliases:     []string{"f"},
+			Usage:       "filename of the cri-o hook that will be created / removed in the hooks directory",
+			Value:       defaultHookFilename,
+			Destination: &hookFilenameFlag,
+			EnvVars:     []string{"CRIO_HOOK_FILENAME"},
+			DefaultText: defaultHookFilename,
+		},
+	}
+
+	// Update the subcommand flags with the common subcommand flags
+	setup.Flags = append([]cli.Flag{}, commonFlags...)
+	cleanup.Flags = append([]cli.Flag{}, commonFlags...)
+
+	// Run the top-level CLI
+	if err := c.Run(os.Args); err != nil {
+		log.Fatal(fmt.Errorf("error: %v", err))
+	}
+}
+
+// Setup installs the prestart hook required to launch GPU-enabled containers
+func Setup(c *cli.Context) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	err := os.MkdirAll(hooksDirFlag, 0755)
+	if err != nil {
+		return fmt.Errorf("error creating hooks directory %v: %v", hooksDirFlag, err)
+	}
+
+	hookPath := getHookPath(hooksDirFlag, hookFilenameFlag)
+	err = createHook(tooklitDirArg, hookPath)
+	if err != nil {
+		return fmt.Errorf("error creating hook: %v", err)
+	}
+
+	return nil
+}
+
+// Cleanup removes the specified prestart hook
+func Cleanup(c *cli.Context) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	hookPath := getHookPath(hooksDirFlag, hookFilenameFlag)
+	err := os.Remove(hookPath)
+	if err != nil {
+		return fmt.Errorf("error removing hook '%v': %v", hookPath, err)
+	}
+
+	return nil
+}
+
+// ParseArgs parses the command line arguments to the CLI
+func ParseArgs(c *cli.Context) error {
+	args := c.Args()
+
+	log.Infof("Parsing arguments: %v", args.Slice())
+	if c.NArg() != 1 {
+		return fmt.Errorf("incorrect number of arguments")
+	}
+	tooklitDirArg = args.Get(0)
+	log.Infof("Successfully parsed arguments")
+
+	return nil
+}
+
+func createHook(toolkitDir string, hookPath string) error {
+	hook, err := os.Create(hookPath)
+	if err != nil {
+		return fmt.Errorf("error creating hook file '%v': %v", hookPath, err)
+	}
+	defer hook.Close()
+
+	encoder := json.NewEncoder(hook)
+	err = encoder.Encode(generateOciHook(tooklitDirArg))
+	if err != nil {
+		return fmt.Errorf("error writing hook file '%v': %v", hookPath, err)
+	}
+	return nil
+}
+
+func getHookPath(hooksDir string, hookFilename string) string {
+	return filepath.Join(hooksDir, hookFilename)
+}
+
+func generateOciHook(toolkitDir string) hooks.Hook {
+	hookPath := filepath.Join(toolkitDir, "nvidia-container-toolkit")
+	envPath := "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:" + toolkitDir
+	always := true
+
+	hook := hooks.Hook{
+		Version: "1.0.0",
+		Stages:  []string{"prestart"},
+		Hook: rspec.Hook{
+			Path: hookPath,
+			Args: []string{"nvidia-container-toolkit", "prestart"},
+			Env:  []string{envPath},
+		},
+		When: hooks.When{
+			Always:   &always,
+			Commands: []string{".*"},
+		},
+	}
+	return hook
+}

tools/container/docker/docker.go

@@ -0,0 +1,462 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+)
+
+const (
+	nvidiaRuntimeName               = "nvidia"
+	nvidiaRuntimeBinary             = "nvidia-container-runtime"
+	nvidiaExperimentalRuntimeName   = "nvidia-experimental"
+	nvidiaExperimentalRuntimeBinary = "nvidia-container-runtime-experimental"
+
+	defaultConfig       = "/etc/docker/daemon.json"
+	defaultSocket       = "/var/run/docker.sock"
+	defaultSetAsDefault = true
+	// defaultRuntimeName specifies the NVIDIA runtime to be use as the default runtime if setting the default runtime is enabled
+	defaultRuntimeName = nvidiaRuntimeName
+
+	reloadBackoff     = 5 * time.Second
+	maxReloadAttempts = 6
+
+	defaultDockerRuntime  = "runc"
+	socketMessageToGetPID = "GET /info HTTP/1.0\r\n\r\n"
+)
+
+// nvidiaRuntimeBinaries defines a map of runtime names to binary names
+var nvidiaRuntimeBinaries = map[string]string{
+	nvidiaRuntimeName:             nvidiaRuntimeBinary,
+	nvidiaExperimentalRuntimeName: nvidiaExperimentalRuntimeBinary,
+}
+
+// options stores the configuration from the command line or environment variables
+type options struct {
+	config       string
+	socket       string
+	runtimeName  string
+	setAsDefault bool
+	runtimeDir   string
+}
+
+func main() {
+	options := options{}
+
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "docker"
+	c.Usage = "Update docker config with the nvidia runtime"
+	c.Version = "0.1.0"
+
+	// Create the 'setup' subcommand
+	setup := cli.Command{}
+	setup.Name = "setup"
+	setup.Usage = "Trigger docker config to be updated"
+	setup.ArgsUsage = "<runtime_dirname>"
+	setup.Action = func(c *cli.Context) error {
+		return Setup(c, &options)
+	}
+
+	// Create the 'cleanup' subcommand
+	cleanup := cli.Command{}
+	cleanup.Name = "cleanup"
+	cleanup.Usage = "Trigger any updates made to docker config to be undone"
+	cleanup.ArgsUsage = "<runtime_dirname>"
+	cleanup.Action = func(c *cli.Context) error {
+		return Cleanup(c, &options)
+	}
+
+	// Register the subcommands with the top-level CLI
+	c.Commands = []*cli.Command{
+		&setup,
+		&cleanup,
+	}
+
+	// Setup common flags across both subcommands. All subcommands get the same
+	// set of flags even if they don't use some of them. This is so that we
+	// only require the user to specify one set of flags for both 'startup'
+	// and 'cleanup' to simplify things.
+	commonFlags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "config",
+			Aliases:     []string{"c"},
+			Usage:       "Path to docker config file",
+			Value:       defaultConfig,
+			Destination: &options.config,
+			EnvVars:     []string{"DOCKER_CONFIG"},
+		},
+		&cli.StringFlag{
+			Name:        "socket",
+			Aliases:     []string{"s"},
+			Usage:       "Path to the docker socket file",
+			Value:       defaultSocket,
+			Destination: &options.socket,
+			EnvVars:     []string{"DOCKER_SOCKET"},
+		},
+		// The flags below are only used by the 'setup' command.
+		&cli.StringFlag{
+			Name:        "runtime-name",
+			Aliases:     []string{"r"},
+			Usage:       "Specify the name of the `nvidia` runtime. If set-as-default is selected, the runtime is used as the default runtime.",
+			Value:       defaultRuntimeName,
+			Destination: &options.runtimeName,
+			EnvVars:     []string{"DOCKER_RUNTIME_NAME"},
+		},
+		&cli.BoolFlag{
+			Name:        "set-as-default",
+			Aliases:     []string{"d"},
+			Usage:       "Set the `nvidia` runtime as the default runtime. If --runtime-name is specified as `nvidia-experimental` the experimental runtime is set as the default runtime instead",
+			Value:       defaultSetAsDefault,
+			Destination: &options.setAsDefault,
+			EnvVars:     []string{"DOCKER_SET_AS_DEFAULT"},
+			Hidden:      true,
+		},
+	}
+
+	// Update the subcommand flags with the common subcommand flags
+	setup.Flags = append([]cli.Flag{}, commonFlags...)
+	cleanup.Flags = append([]cli.Flag{}, commonFlags...)
+
+	// Run the top-level CLI
+	if err := c.Run(os.Args); err != nil {
+		log.Errorf("Error running docker configuration: %v", err)
+		os.Exit(1)
+	}
+}
+
+// Setup updates docker configuration to include the nvidia runtime and reloads it
+func Setup(c *cli.Context, o *options) error {
+	log.Infof("Starting 'setup' for %v", c.App.Name)
+
+	runtimeDir, err := ParseArgs(c)
+	if err != nil {
+		return fmt.Errorf("unable to parse args: %v", err)
+	}
+	o.runtimeDir = runtimeDir
+
+	cfg, err := LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = UpdateConfig(cfg, o)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = FlushConfig(cfg, o.config)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = SignalDocker(o.socket)
+	if err != nil {
+		return fmt.Errorf("unable to signal docker: %v", err)
+	}
+
+	log.Infof("Completed 'setup' for %v", c.App.Name)
+
+	return nil
+}
+
+// Cleanup reverts docker configuration to remove the nvidia runtime and reloads it
+func Cleanup(c *cli.Context, o *options) error {
+	log.Infof("Starting 'cleanup' for %v", c.App.Name)
+
+	_, err := ParseArgs(c)
+	if err != nil {
+		return fmt.Errorf("unable to parse args: %v", err)
+	}
+
+	cfg, err := LoadConfig(o.config)
+	if err != nil {
+		return fmt.Errorf("unable to load config: %v", err)
+	}
+
+	err = RevertConfig(cfg)
+	if err != nil {
+		return fmt.Errorf("unable to update config: %v", err)
+	}
+
+	err = FlushConfig(cfg, o.config)
+	if err != nil {
+		return fmt.Errorf("unable to flush config: %v", err)
+	}
+
+	err = SignalDocker(o.socket)
+	if err != nil {
+		return fmt.Errorf("unable to signal docker: %v", err)
+	}
+
+	log.Infof("Completed 'cleanup' for %v", c.App.Name)
+
+	return nil
+}
+
+// ParseArgs parses the command line arguments to the CLI
+func ParseArgs(c *cli.Context) (string, error) {
+	args := c.Args()
+
+	log.Infof("Parsing arguments: %v", args.Slice())
+	if args.Len() != 1 {
+		return "", fmt.Errorf("incorrect number of arguments")
+	}
+	runtimeDir := args.Get(0)
+	log.Infof("Successfully parsed arguments")
+
+	return runtimeDir, nil
+}
+
+// LoadConfig loads the docker config from disk
+func LoadConfig(config string) (map[string]interface{}, error) {
+	log.Infof("Loading config: %v", config)
+
+	info, err := os.Stat(config)
+	if os.IsExist(err) && info.IsDir() {
+		return nil, fmt.Errorf("config file is a directory")
+	}
+
+	cfg := make(map[string]interface{})
+
+	if os.IsNotExist(err) {
+		log.Infof("Config file does not exist, creating new one")
+		return cfg, nil
+	}
+
+	readBytes, err := ioutil.ReadFile(config)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read config: %v", err)
+	}
+
+	reader := bytes.NewReader(readBytes)
+	if err := json.NewDecoder(reader).Decode(&cfg); err != nil {
+		return nil, err
+	}
+
+	log.Infof("Successfully loaded config")
+	return cfg, nil
+}
+
+// UpdateConfig updates the docker config to include the nvidia runtimes
+func UpdateConfig(config map[string]interface{}, o *options) error {
+	defaultRuntime := o.getDefaultRuntime()
+	if defaultRuntime != "" {
+		config["default-runtime"] = defaultRuntime
+	}
+
+	runtimes := make(map[string]interface{})
+	if _, exists := config["runtimes"]; exists {
+		runtimes = config["runtimes"].(map[string]interface{})
+	}
+
+	for name, rt := range o.runtimes() {
+		runtimes[name] = rt
+	}
+
+	config["runtimes"] = runtimes
+	return nil
+}
+
+//RevertConfig reverts the docker config to remove the nvidia runtime
+func RevertConfig(config map[string]interface{}) error {
+	if _, exists := config["default-runtime"]; exists {
+		defaultRuntime := config["default-runtime"].(string)
+		if _, exists := nvidiaRuntimeBinaries[defaultRuntime]; exists {
+			config["default-runtime"] = defaultDockerRuntime
+		}
+	}
+
+	if _, exists := config["runtimes"]; exists {
+		runtimes := config["runtimes"].(map[string]interface{})
+
+		for name := range nvidiaRuntimeBinaries {
+			delete(runtimes, name)
+		}
+
+		if len(runtimes) == 0 {
+			delete(config, "runtimes")
+		}
+	}
+	return nil
+}
+
+// FlushConfig flushes the updated/reverted config out to disk
+func FlushConfig(cfg map[string]interface{}, config string) error {
+	log.Infof("Flushing config")
+
+	output, err := json.MarshalIndent(cfg, "", "    ")
+	if err != nil {
+		return fmt.Errorf("unable to convert to JSON: %v", err)
+	}
+
+	switch len(output) {
+	case 0:
+		err := os.Remove(config)
+		if err != nil {
+			return fmt.Errorf("unable to remove empty file: %v", err)
+		}
+		log.Infof("Config empty, removing file")
+	default:
+		f, err := os.Create(config)
+		if err != nil {
+			return fmt.Errorf("unable to open %v for writing: %v", config, err)
+		}
+		defer f.Close()
+
+		_, err = f.WriteString(string(output))
+		if err != nil {
+			return fmt.Errorf("unable to write output: %v", err)
+		}
+	}
+
+	log.Infof("Successfully flushed config")
+
+	return nil
+}
+
+// SignalDocker sends a SIGHUP signal to docker daemon
+func SignalDocker(socket string) error {
+	log.Infof("Sending SIGHUP signal to docker")
+
+	// Wrap the logic to perform the SIGHUP in a function so we can retry it on failure
+	retriable := func() error {
+		conn, err := net.Dial("unix", socket)
+		if err != nil {
+			return fmt.Errorf("unable to dial: %v", err)
+		}
+		defer conn.Close()
+
+		sconn, err := conn.(*net.UnixConn).SyscallConn()
+		if err != nil {
+			return fmt.Errorf("unable to get syscall connection: %v", err)
+		}
+
+		err1 := sconn.Control(func(fd uintptr) {
+			err = syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, syscall.SO_PASSCRED, 1)
+		})
+		if err1 != nil {
+			return fmt.Errorf("unable to issue call on socket fd: %v", err1)
+		}
+		if err != nil {
+			return fmt.Errorf("unable to SetsockoptInt on socket fd: %v", err)
+		}
+
+		_, _, err = conn.(*net.UnixConn).WriteMsgUnix([]byte(socketMessageToGetPID), nil, nil)
+		if err != nil {
+			return fmt.Errorf("unable to WriteMsgUnix on socket fd: %v", err)
+		}
+
+		oob := make([]byte, 1024)
+		_, oobn, _, _, err := conn.(*net.UnixConn).ReadMsgUnix(nil, oob)
+		if err != nil {
+			return fmt.Errorf("unable to ReadMsgUnix on socket fd: %v", err)
+		}
+
+		oob = oob[:oobn]
+		scm, err := syscall.ParseSocketControlMessage(oob)
+		if err != nil {
+			return fmt.Errorf("unable to ParseSocketControlMessage from message received on socket fd: %v", err)
+		}
+
+		ucred, err := syscall.ParseUnixCredentials(&scm[0])
+		if err != nil {
+			return fmt.Errorf("unable to ParseUnixCredentials from message received on socket fd: %v", err)
+		}
+
+		err = syscall.Kill(int(ucred.Pid), syscall.SIGHUP)
+		if err != nil {
+			return fmt.Errorf("unable to send SIGHUP to 'docker' process: %v", err)
+		}
+
+		return nil
+	}
+
+	// Try to send a SIGHUP up to maxReloadAttempts times
+	var err error
+	for i := 0; i < maxReloadAttempts; i++ {
+		err = retriable()
+		if err == nil {
+			break
+		}
+		if i == maxReloadAttempts-1 {
+			break
+		}
+		log.Warnf("Error signaling docker, attempt %v/%v: %v", i+1, maxReloadAttempts, err)
+		time.Sleep(reloadBackoff)
+	}
+	if err != nil {
+		log.Warnf("Max retries reached %v/%v, aborting", maxReloadAttempts, maxReloadAttempts)
+		return err
+	}
+
+	log.Infof("Successfully signaled docker")
+
+	return nil
+}
+
+// getDefaultRuntime returns the default runtime for the configured options.
+// If the configuration is invalid or the default runtimes should not be set
+// the empty string is returned.
+func (o options) getDefaultRuntime() string {
+	if o.setAsDefault == false {
+		return ""
+	}
+
+	return o.runtimeName
+}
+
+// runtimes returns the docker runtime definitions for the supported nvidia runtimes
+// for the given options. This includes the path with the options runtimeDir applied
+func (o options) runtimes() map[string]interface{} {
+	runtimes := make(map[string]interface{})
+	for r, bin := range o.getRuntimeBinaries() {
+		runtimes[r] = map[string]interface{}{
+			"path": bin,
+			"args": []string{},
+		}
+	}
+	return runtimes
+}
+
+// getRuntimeBinaries returns a map of runtime names to binary paths. This includes the
+// renaming of the `nvidia` runtime as per the --runtime-class command line flag.
+func (o options) getRuntimeBinaries() map[string]string {
+	runtimeBinaries := make(map[string]string)
+
+	for rt, bin := range nvidiaRuntimeBinaries {
+		runtime := rt
+		if o.runtimeName != "" && o.runtimeName != nvidiaExperimentalRuntimeName && runtime == defaultRuntimeName {
+			runtime = o.runtimeName
+		}
+
+		runtimeBinaries[runtime] = filepath.Join(o.runtimeDir, bin)
+	}
+
+	return runtimeBinaries
+}

tools/container/docker/docker_test.go

@@ -0,0 +1,423 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"encoding/json"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestUpdateConfigDefaultRuntime(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		setAsDefault               bool
+		runtimeName                string
+		expectedDefaultRuntimeName interface{}
+	}{
+		{},
+		{
+			setAsDefault:               false,
+			expectedDefaultRuntimeName: nil,
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "NAME",
+			expectedDefaultRuntimeName: "NAME",
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "nvidia-experimental",
+			expectedDefaultRuntimeName: "nvidia-experimental",
+		},
+		{
+			setAsDefault:               true,
+			runtimeName:                "nvidia",
+			expectedDefaultRuntimeName: "nvidia",
+		},
+	}
+
+	for i, tc := range testCases {
+		o := &options{
+			setAsDefault: tc.setAsDefault,
+			runtimeName:  tc.runtimeName,
+			runtimeDir:   runtimeDir,
+		}
+
+		config := map[string]interface{}{}
+
+		err := UpdateConfig(config, o)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		defaultRuntimeName := config["default-runtime"]
+		require.EqualValues(t, tc.expectedDefaultRuntimeName, defaultRuntimeName, "%d: %v", i, tc)
+	}
+}
+
+func TestUpdateConfig(t *testing.T) {
+	const runtimeDir = "/test/runtime/dir"
+
+	testCases := []struct {
+		config         map[string]interface{}
+		setAsDefault   bool
+		runtimeName    string
+		expectedConfig map[string]interface{}
+	}{
+		{
+			config:       map[string]interface{}{},
+			setAsDefault: false,
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config:       map[string]interface{}{},
+			setAsDefault: false,
+			runtimeName:  "NAME",
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"NAME": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config:       map[string]interface{}{},
+			setAsDefault: false,
+			runtimeName:  "nvidia-experimental",
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			setAsDefault: false,
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"not-nvidia": map[string]interface{}{
+						"path": "some-other-path",
+						"args": []string{},
+					},
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "runc",
+			},
+			setAsDefault: true,
+			runtimeName:  "nvidia",
+			expectedConfig: map[string]interface{}{
+				"default-runtime": "nvidia",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "runc",
+			},
+			setAsDefault: true,
+			runtimeName:  "nvidia-experimental",
+			expectedConfig: map[string]interface{}{
+				"default-runtime": "nvidia-experimental",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		options := &options{
+			setAsDefault: tc.setAsDefault,
+			runtimeName:  tc.runtimeName,
+			runtimeDir:   runtimeDir,
+		}
+		err := UpdateConfig(tc.config, options)
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		configContent, err := json.MarshalIndent(tc.config, "", "    ")
+		require.NoError(t, err)
+
+		expectedContent, err := json.MarshalIndent(tc.expectedConfig, "", "    ")
+		require.NoError(t, err)
+
+		require.EqualValues(t, string(expectedContent), string(configContent), "%d: %v", i, tc)
+	}
+}
+
+func TestRevertConfig(t *testing.T) {
+	testCases := []struct {
+		config         map[string]interface{}
+		expectedConfig map[string]interface{}
+	}{
+		{
+			config:         map[string]interface{}{},
+			expectedConfig: map[string]interface{}{},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{},
+		},
+		{
+			config: map[string]interface{}{
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+					"nvidia-experimental": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime-experimental",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{},
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "nvidia",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"default-runtime": "runc",
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"default-runtime": "not-nvidia",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"default-runtime": "not-nvidia",
+			},
+		},
+		{
+			config: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+				"runtimes": map[string]interface{}{
+					"nvidia": map[string]interface{}{
+						"path": "/test/runtime/dir/nvidia-container-runtime",
+						"args": []string{},
+					},
+				},
+			},
+			expectedConfig: map[string]interface{}{
+				"exec-opts":  []string{"native.cgroupdriver=systemd"},
+				"log-driver": "json-file",
+				"log-opts": map[string]string{
+					"max-size": "100m",
+				},
+				"storage-driver": "overlay2",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		err := RevertConfig(tc.config)
+
+		require.NoError(t, err, "%d: %v", i, tc)
+
+		configContent, err := json.MarshalIndent(tc.config, "", "    ")
+		require.NoError(t, err)
+
+		expectedContent, err := json.MarshalIndent(tc.expectedConfig, "", "    ")
+		require.NoError(t, err)
+
+		require.EqualValues(t, string(expectedContent), string(configContent), "%d: %v", i, tc)
+	}
+}
+
+func TestFlagsDefaultRuntime(t *testing.T) {
+	testCases := []struct {
+		setAsDefault bool
+		runtimeName  string
+		expected     string
+	}{
+		{
+			expected: "",
+		},
+		{
+			runtimeName: "not-bool",
+			expected:    "",
+		},
+		{
+			setAsDefault: false,
+			runtimeName:  "nvidia",
+			expected:     "",
+		},
+		{
+			setAsDefault: true,
+			runtimeName:  "nvidia",
+			expected:     "nvidia",
+		},
+		{
+			setAsDefault: true,
+			runtimeName:  "nvidia-experimental",
+			expected:     "nvidia-experimental",
+		},
+	}
+
+	for i, tc := range testCases {
+		f := options{
+			setAsDefault: tc.setAsDefault,
+			runtimeName:  tc.runtimeName,
+		}
+
+		require.Equal(t, tc.expected, f.getDefaultRuntime(), "%d: %v", i, tc)
+	}
+}

tools/container/nvidia-toolkit/run.go

@@ -0,0 +1,290 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"os/signal"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	log "github.com/sirupsen/logrus"
+	cli "github.com/urfave/cli/v2"
+	unix "golang.org/x/sys/unix"
+)
+
+const (
+	runDir         = "/run/nvidia"
+	pidFile        = runDir + "/toolkit.pid"
+	toolkitCommand = "toolkit"
+	toolkitSubDir  = "toolkit"
+
+	defaultToolkitArgs = ""
+	defaultRuntime     = "docker"
+	defaultRuntimeArgs = ""
+)
+
+var availableRuntimes = map[string]struct{}{"docker": {}, "crio": {}, "containerd": {}}
+
+var waitingForSignal = make(chan bool, 1)
+var signalReceived = make(chan bool, 1)
+
+var destinationArg string
+var noDaemonFlag bool
+var toolkitArgsFlag string
+var runtimeFlag string
+var runtimeArgsFlag string
+
+// Version defines the CLI version. This is set at build time using LD FLAGS
+var Version = "development"
+
+func main() {
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "nvidia-toolkit"
+	c.Usage = "Install the nvidia-container-toolkit for use by a given runtime"
+	c.UsageText = "DESTINATION [-n | --no-daemon] [-t | --toolkit-args] [-r | --runtime] [-u | --runtime-args]"
+	c.Description = "DESTINATION points to the host path underneath which the nvidia-container-toolkit should be installed.\nIt will be installed at ${DESTINATION}/toolkit"
+	c.Version = Version
+	c.Action = Run
+
+	// Setup flags for the CLI
+	c.Flags = []cli.Flag{
+		&cli.BoolFlag{
+			Name:        "no-daemon",
+			Aliases:     []string{"n"},
+			Usage:       "terminate immediatly after setting up the runtime. Note that no cleanup will be performed",
+			Destination: &noDaemonFlag,
+			EnvVars:     []string{"NO_DAEMON"},
+		},
+		&cli.StringFlag{
+			Name:        "toolkit-args",
+			Aliases:     []string{"t"},
+			Usage:       "arguments to pass to the underlying 'toolkit' command",
+			Value:       defaultToolkitArgs,
+			Destination: &toolkitArgsFlag,
+			EnvVars:     []string{"TOOLKIT_ARGS"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime",
+			Aliases:     []string{"r"},
+			Usage:       "the runtime to setup on this node. One of {'docker', 'crio', 'containerd'}",
+			Value:       defaultRuntime,
+			Destination: &runtimeFlag,
+			EnvVars:     []string{"RUNTIME"},
+		},
+		&cli.StringFlag{
+			Name:        "runtime-args",
+			Aliases:     []string{"u"},
+			Usage:       "arguments to pass to 'docker', 'crio', or 'containerd' setup command",
+			Value:       defaultRuntimeArgs,
+			Destination: &runtimeArgsFlag,
+			EnvVars:     []string{"RUNTIME_ARGS"},
+		},
+	}
+
+	// Run the CLI
+	log.Infof("Starting %v", c.Name)
+
+	remainingArgs, err := ParseArgs(os.Args)
+	if err != nil {
+		log.Errorf("Error: unable to parse arguments: %v", err)
+		os.Exit(1)
+	}
+
+	if err := c.Run(remainingArgs); err != nil {
+		log.Errorf("error running nvidia-toolkit: %v", err)
+		os.Exit(1)
+	}
+
+	log.Infof("Completed %v", c.Name)
+}
+
+// Run runs the core logic of the CLI
+func Run(c *cli.Context) error {
+	err := verifyFlags()
+	if err != nil {
+		return fmt.Errorf("unable to verify flags: %v", err)
+	}
+
+	err = initialize()
+	if err != nil {
+		return fmt.Errorf("unable to initialize: %v", err)
+	}
+	defer shutdown()
+
+	err = installToolkit()
+	if err != nil {
+		return fmt.Errorf("unable to install toolkit: %v", err)
+	}
+
+	err = setupRuntime()
+	if err != nil {
+		return fmt.Errorf("unable to setup runtime: %v", err)
+	}
+
+	if !noDaemonFlag {
+		err = waitForSignal()
+		if err != nil {
+			return fmt.Errorf("unable to wait for signal: %v", err)
+		}
+
+		err = cleanupRuntime()
+		if err != nil {
+			return fmt.Errorf("unable to cleanup runtime: %v", err)
+		}
+	}
+
+	return nil
+}
+
+// ParseArgs parses the command line arguments and returns the remaining arguments
+func ParseArgs(args []string) ([]string, error) {
+	log.Infof("Parsing arguments")
+
+	numPositionalArgs := 2 // Includes command itself
+
+	if len(args) < numPositionalArgs {
+		return nil, fmt.Errorf("missing arguments")
+	}
+
+	for _, arg := range args {
+		if arg == "--help" || arg == "-h" {
+			return []string{args[0], arg}, nil
+		}
+		if arg == "--version" || arg == "-v" {
+			return []string{args[0], arg}, nil
+		}
+	}
+
+	for _, arg := range args[:numPositionalArgs] {
+		if strings.HasPrefix(arg, "-") {
+			return nil, fmt.Errorf("unexpected flag where argument should be")
+		}
+	}
+
+	for _, arg := range args[numPositionalArgs:] {
+		if !strings.HasPrefix(arg, "-") {
+			return nil, fmt.Errorf("unexpected argument where flag should be")
+		}
+	}
+
+	destinationArg = args[1]
+
+	return append([]string{args[0]}, args[numPositionalArgs:]...), nil
+}
+
+func verifyFlags() error {
+	log.Infof("Verifying Flags")
+	if _, exists := availableRuntimes[runtimeFlag]; !exists {
+		return fmt.Errorf("unknown runtime: %v", runtimeFlag)
+	}
+	return nil
+}
+
+func initialize() error {
+	log.Infof("Initializing")
+
+	f, err := os.Create(pidFile)
+	if err != nil {
+		return fmt.Errorf("unable to create pidfile: %v", err)
+	}
+
+	err = unix.Flock(int(f.Fd()), unix.LOCK_EX|unix.LOCK_NB)
+	if err != nil {
+		log.Warnf("Unable to get exclusive lock on '%v'", pidFile)
+		log.Warnf("This normally means an instance of the NVIDIA toolkit Container is already running, aborting")
+		return fmt.Errorf("unable to get flock on pidfile: %v", err)
+	}
+
+	_, err = f.WriteString(fmt.Sprintf("%v\n", os.Getpid()))
+	if err != nil {
+		return fmt.Errorf("unable to write PID to pidfile: %v", err)
+	}
+
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGQUIT, syscall.SIGPIPE, syscall.SIGTERM)
+	go func() {
+		<-sigs
+		select {
+		case <-waitingForSignal:
+			signalReceived <- true
+		default:
+			log.Infof("Signal received, exiting early")
+			shutdown()
+			os.Exit(0)
+		}
+	}()
+
+	return nil
+}
+
+func installToolkit() error {
+	toolkitDir := filepath.Join(destinationArg, toolkitSubDir)
+
+	log.Infof("Installing toolkit")
+
+	cmdline := fmt.Sprintf("%v install %v %v\n", toolkitCommand, toolkitArgsFlag, toolkitDir)
+	cmd := exec.Command("sh", "-c", cmdline)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error running %v command: %v", toolkitCommand, err)
+	}
+
+	return nil
+}
+
+func setupRuntime() error {
+	toolkitDir := filepath.Join(destinationArg, toolkitSubDir)
+
+	log.Infof("Setting up runtime")
+
+	cmdline := fmt.Sprintf("%v setup %v %v\n", runtimeFlag, runtimeArgsFlag, toolkitDir)
+
+	cmd := exec.Command("sh", "-c", cmdline)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error running %v command: %v", runtimeFlag, err)
+	}
+
+	return nil
+}
+
+func waitForSignal() error {
+	log.Infof("Waiting for signal")
+	waitingForSignal <- true
+	<-signalReceived
+	return nil
+}
+
+func cleanupRuntime() error {
+	toolkitDir := filepath.Join(destinationArg, toolkitSubDir)
+
+	log.Infof("Cleaning up Runtime")
+
+	cmdline := fmt.Sprintf("%v cleanup %v %v\n", runtimeFlag, runtimeArgsFlag, toolkitDir)
+
+	cmd := exec.Command("sh", "-c", cmdline)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	err := cmd.Run()
+	if err != nil {
+		return fmt.Errorf("error running %v command: %v", runtimeFlag, err)
+	}
+
+	return nil
+}
+
+func shutdown() {
+	log.Infof("Shutting Down")
+
+	err := os.Remove(pidFile)
+	if err != nil {
+		log.Warnf("Unable to remove pidfile: %v", err)
+	}
+}

tools/container/toolkit/executable.go

@@ -0,0 +1,153 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type executableTarget struct {
+	dotfileName string
+	wrapperName string
+}
+
+type executable struct {
+	source   string
+	target   executableTarget
+	env      map[string]string
+	preLines []string
+	argLines []string
+}
+
+// install installs an executable component of the NVIDIA container toolkit. The source executable
+// is copied to a `.real` file and a wapper is created to set up the environment as required.
+func (e executable) install(destFolder string) (string, error) {
+	log.Infof("Installing executable '%v' to %v", e.source, destFolder)
+
+	dotfileName := e.dotfileName()
+
+	installedDotfileName, err := installFileToFolderWithName(destFolder, dotfileName, e.source)
+	if err != nil {
+		return "", fmt.Errorf("error installing file '%v' as '%v': %v", e.source, dotfileName, err)
+	}
+	log.Infof("Installed '%v'", installedDotfileName)
+
+	wrapperFilename, err := e.installWrapper(destFolder, installedDotfileName)
+	if err != nil {
+		return "", fmt.Errorf("error wrapping '%v': %v", installedDotfileName, err)
+	}
+	log.Infof("Installed wrapper '%v'", wrapperFilename)
+
+	return wrapperFilename, nil
+}
+
+func (e executable) dotfileName() string {
+	return e.target.dotfileName
+}
+
+func (e executable) wrapperName() string {
+	return e.target.wrapperName
+}
+
+func (e executable) installWrapper(destFolder string, dotfileName string) (string, error) {
+	wrapperPath := filepath.Join(destFolder, e.wrapperName())
+	wrapper, err := os.Create(wrapperPath)
+	if err != nil {
+		return "", fmt.Errorf("error creating executable wrapper: %v", err)
+	}
+	defer wrapper.Close()
+
+	err = e.writeWrapperTo(wrapper, destFolder, dotfileName)
+	if err != nil {
+		return "", fmt.Errorf("error writing wrapper contents: %v", err)
+	}
+
+	err = ensureExecutable(wrapperPath)
+	if err != nil {
+		return "", fmt.Errorf("error making wrapper executable: %v", err)
+	}
+	return wrapperPath, nil
+}
+
+func (e executable) writeWrapperTo(wrapper io.Writer, destFolder string, dotfileName string) error {
+	r := newReplacements(destDirPattern, destFolder)
+
+	// Add the shebang
+	fmt.Fprintln(wrapper, "#! /bin/sh")
+
+	// Add the preceding lines if any
+	for _, line := range e.preLines {
+		fmt.Fprintf(wrapper, "%s\n", r.apply(line))
+	}
+
+	// Update the path to include the destination folder
+	var env map[string]string
+	if e.env == nil {
+		env = make(map[string]string)
+	} else {
+		env = e.env
+	}
+
+	path, specified := env["PATH"]
+	if !specified {
+		path = "$PATH"
+	}
+	env["PATH"] = strings.Join([]string{destFolder, path}, ":")
+
+	var sortedEnvvars []string
+	for e := range env {
+		sortedEnvvars = append(sortedEnvvars, e)
+	}
+	sort.Strings(sortedEnvvars)
+
+	for _, e := range sortedEnvvars {
+		v := env[e]
+		fmt.Fprintf(wrapper, "%s=%s \\\n", e, r.apply(v))
+	}
+	// Add the call to the target executable
+	fmt.Fprintf(wrapper, "%s \\\n", dotfileName)
+
+	// Insert additional lines in the `arg` list
+	for _, line := range e.argLines {
+		fmt.Fprintf(wrapper, "\t%s \\\n", r.apply(line))
+	}
+	// Add the script arguments "$@"
+	fmt.Fprintln(wrapper, "\t\"$@\"")
+
+	return nil
+}
+
+// ensureExecutable is equivalent to running chmod +x on the specified file
+func ensureExecutable(path string) error {
+	info, err := os.Stat(path)
+	if err != nil {
+		return fmt.Errorf("error getting file info for '%v': %v", path, err)
+	}
+	executableMode := info.Mode() | 0111
+	err = os.Chmod(path, executableMode)
+	if err != nil {
+		return fmt.Errorf("error setting executable mode for '%v': %v", path, err)
+	}
+	return nil
+}

tools/container/toolkit/executable_test.go

@@ -0,0 +1,152 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestWrapper(t *testing.T) {
+	const shebang = "#! /bin/sh"
+	const destFolder = "/dest/folder"
+	const dotfileName = "source.real"
+
+	testCases := []struct {
+		e             executable
+		expectedLines []string
+	}{
+		{
+			e: executable{},
+			expectedLines: []string{
+				shebang,
+				"PATH=/dest/folder:$PATH \\",
+				"source.real \\",
+				"\t\"$@\"",
+				"",
+			},
+		},
+		{
+			e: executable{
+				env: map[string]string{
+					"PATH": "some-path",
+				},
+			},
+			expectedLines: []string{
+				shebang,
+				"PATH=/dest/folder:some-path \\",
+				"source.real \\",
+				"\t\"$@\"",
+				"",
+			},
+		},
+		{
+			e: executable{
+				preLines: []string{
+					"preline1",
+					"preline2",
+				},
+			},
+			expectedLines: []string{
+				shebang,
+				"preline1",
+				"preline2",
+				"PATH=/dest/folder:$PATH \\",
+				"source.real \\",
+				"\t\"$@\"",
+				"",
+			},
+		},
+		{
+			e: executable{
+				argLines: []string{
+					"argline1",
+					"argline2",
+				},
+			},
+			expectedLines: []string{
+				shebang,
+				"PATH=/dest/folder:$PATH \\",
+				"source.real \\",
+				"\targline1 \\",
+				"\targline2 \\",
+				"\t\"$@\"",
+				"",
+			},
+		},
+	}
+
+	for i, tc := range testCases {
+		buf := &bytes.Buffer{}
+
+		err := tc.e.writeWrapperTo(buf, destFolder, dotfileName)
+		require.NoError(t, err)
+
+		exepectedContents := strings.Join(tc.expectedLines, "\n")
+		require.Equal(t, exepectedContents, buf.String(), "%v: %v", i, tc)
+	}
+}
+
+func TestInstallExecutable(t *testing.T) {
+	inputFolder, err := os.MkdirTemp("", "")
+	require.NoError(t, err)
+	defer os.RemoveAll(inputFolder)
+
+	// Create the source file
+	source := filepath.Join(inputFolder, "input")
+	sourceFile, err := os.Create(source)
+
+	base := filepath.Base(source)
+
+	require.NoError(t, err)
+	require.NoError(t, sourceFile.Close())
+
+	e := executable{
+		source: source,
+		target: executableTarget{
+			dotfileName: "input.real",
+			wrapperName: "input",
+		},
+	}
+
+	destFolder, err := os.MkdirTemp("", "output-*")
+	require.NoError(t, err)
+	defer os.RemoveAll(destFolder)
+
+	installed, err := e.install(destFolder)
+
+	require.NoError(t, err)
+	require.Equal(t, filepath.Join(destFolder, base), installed)
+
+	// Now check the post conditions:
+	sourceInfo, err := os.Stat(source)
+	require.NoError(t, err)
+
+	destInfo, err := os.Stat(filepath.Join(destFolder, base+".real"))
+	require.NoError(t, err)
+	require.Equal(t, sourceInfo.Size(), destInfo.Size())
+	require.Equal(t, sourceInfo.Mode(), destInfo.Mode())
+
+	wrapperInfo, err := os.Stat(installed)
+	require.NoError(t, err)
+	require.NotEqual(t, 0, wrapperInfo.Mode()&0111)
+}

tools/container/toolkit/replacements.go

@@ -0,0 +1,45 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import "strings"
+
+const (
+	destDirPattern = "@destDir@"
+)
+
+type replacements map[string]string
+
+func newReplacements(rules ...string) replacements {
+	r := make(replacements)
+	for i := 0; i < len(rules)-1; i += 2 {
+		old := rules[i]
+		new := rules[i+1]
+
+		r[old] = new
+	}
+
+	return r
+}
+
+func (r replacements) apply(input string) string {
+	output := input
+	for old, new := range r {
+		output = strings.ReplaceAll(output, old, new)
+	}
+	return output
+}

tools/container/toolkit/runtime.go

@@ -0,0 +1,132 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	nvidiaContainerRuntimeSource  = "/usr/bin/nvidia-container-runtime"
+	nvidiaContainerRuntimeTarget  = "nvidia-container-runtime.real"
+	nvidiaContainerRuntimeWrapper = "nvidia-container-runtime"
+
+	nvidiaExperimentalContainerRuntimeSource  = "nvidia-container-runtime.experimental"
+	nvidiaExperimentalContainerRuntimeTarget  = nvidiaExperimentalContainerRuntimeSource
+	nvidiaExperimentalContainerRuntimeWrapper = "nvidia-container-runtime-experimental"
+)
+
+// installContainerRuntimes sets up the NVIDIA container runtimes, copying the executables
+// and implementing the required wrapper
+func installContainerRuntimes(toolkitDir string, driverRoot string) error {
+	r := newNvidiaContainerRuntimeInstaller()
+
+	_, err := r.install(toolkitDir)
+	if err != nil {
+		return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
+	}
+
+	// Install the experimental runtime and treat failures as non-fatal.
+	err = installExperimentalRuntime(toolkitDir, driverRoot)
+	if err != nil {
+		log.Warnf("Could not install experimental runtime: %v", err)
+	}
+
+	return nil
+}
+
+// installExperimentalRuntime ensures that the experimental NVIDIA Container runtime is installed
+func installExperimentalRuntime(toolkitDir string, driverRoot string) error {
+	libraryRoot, err := findLibraryRoot(driverRoot)
+	if err != nil {
+		log.Warnf("Error finding library path for root %v: %v", driverRoot, err)
+	}
+	log.Infof("Using library root %v", libraryRoot)
+
+	e := newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot)
+	_, err = e.install(toolkitDir)
+	if err != nil {
+		return fmt.Errorf("error installing experimental NVIDIA Container Runtime: %v", err)
+	}
+
+	return nil
+}
+
+func newNvidiaContainerRuntimeInstaller() *executable {
+	target := executableTarget{
+		dotfileName: nvidiaContainerRuntimeTarget,
+		wrapperName: nvidiaContainerRuntimeWrapper,
+	}
+	return newRuntimeInstaller(nvidiaContainerRuntimeSource, target, nil)
+}
+
+func newNvidiaContainerRuntimeExperimentalInstaller(libraryRoot string) *executable {
+	target := executableTarget{
+		dotfileName: nvidiaExperimentalContainerRuntimeTarget,
+		wrapperName: nvidiaExperimentalContainerRuntimeWrapper,
+	}
+
+	env := make(map[string]string)
+	if libraryRoot != "" {
+		env["LD_LIBRARY_PATH"] = strings.Join([]string{libraryRoot, "$LD_LIBRARY_PATH"}, ":")
+	}
+	return newRuntimeInstaller(nvidiaExperimentalContainerRuntimeSource, target, env)
+}
+
+func newRuntimeInstaller(source string, target executableTarget, env map[string]string) *executable {
+	preLines := []string{
+		"",
+		"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
+		"if [ \"${?}\" != \"0\" ]; then",
+		"	echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
+		"	exec runc \"$@\"",
+		"fi",
+		"",
+	}
+
+	runtimeEnv := make(map[string]string)
+	runtimeEnv["XDG_CONFIG_HOME"] = filepath.Join(destDirPattern, ".config")
+	for k, v := range env {
+		runtimeEnv[k] = v
+	}
+
+	r := executable{
+		source:   source,
+		target:   target,
+		env:      runtimeEnv,
+		preLines: preLines,
+	}
+
+	return &r
+}
+
+func findLibraryRoot(root string) (string, error) {
+	libnvidiamlPath, err := findManagementLibrary(root)
+	if err != nil {
+		return "", fmt.Errorf("error locating NVIDIA management library: %v", err)
+	}
+
+	return filepath.Dir(libnvidiamlPath), nil
+}
+
+func findManagementLibrary(root string) (string, error) {
+	return findLibrary(root, "libnvidia-ml.so")
+}

tools/container/toolkit/runtime_test.go

@@ -0,0 +1,90 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"bytes"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestNvidiaContainerRuntimeInstallerWrapper(t *testing.T) {
+	r := newNvidiaContainerRuntimeInstaller()
+
+	const shebang = "#! /bin/sh"
+	const destFolder = "/dest/folder"
+	const dotfileName = "source.real"
+
+	buf := &bytes.Buffer{}
+
+	err := r.writeWrapperTo(buf, destFolder, dotfileName)
+	require.NoError(t, err)
+
+	expectedLines := []string{
+		shebang,
+		"",
+		"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
+		"if [ \"${?}\" != \"0\" ]; then",
+		"	echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
+		"	exec runc \"$@\"",
+		"fi",
+		"",
+		"PATH=/dest/folder:$PATH \\",
+		"XDG_CONFIG_HOME=/dest/folder/.config \\",
+		"source.real \\",
+		"\t\"$@\"",
+		"",
+	}
+
+	exepectedContents := strings.Join(expectedLines, "\n")
+	require.Equal(t, exepectedContents, buf.String())
+}
+
+func TestExperimentalContainerRuntimeInstallerWrapper(t *testing.T) {
+	r := newNvidiaContainerRuntimeExperimentalInstaller("/some/root/usr/lib64")
+
+	const shebang = "#! /bin/sh"
+	const destFolder = "/dest/folder"
+	const dotfileName = "source.real"
+
+	buf := &bytes.Buffer{}
+
+	err := r.writeWrapperTo(buf, destFolder, dotfileName)
+	require.NoError(t, err)
+
+	expectedLines := []string{
+		shebang,
+		"",
+		"cat /proc/modules | grep -e \"^nvidia \" >/dev/null 2>&1",
+		"if [ \"${?}\" != \"0\" ]; then",
+		"	echo \"nvidia driver modules are not yet loaded, invoking runc directly\"",
+		"	exec runc \"$@\"",
+		"fi",
+		"",
+		"LD_LIBRARY_PATH=/some/root/usr/lib64:$LD_LIBRARY_PATH \\",
+		"PATH=/dest/folder:$PATH \\",
+		"XDG_CONFIG_HOME=/dest/folder/.config \\",
+		"source.real \\",
+		"\t\"$@\"",
+		"",
+	}
+
+	exepectedContents := strings.Join(expectedLines, "\n")
+	require.Equal(t, exepectedContents, buf.String())
+}

tools/container/toolkit/toolkit.go

@@ -0,0 +1,449 @@
+/**
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+*/
+
+package main
+
+import (
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	toml "github.com/pelletier/go-toml"
+	log "github.com/sirupsen/logrus"
+	"github.com/urfave/cli/v2"
+)
+
+const (
+	// DefaultNvidiaDriverRoot specifies the default NVIDIA driver run directory
+	DefaultNvidiaDriverRoot = "/run/nvidia/driver"
+
+	nvidiaContainerCliSource         = "/usr/bin/nvidia-container-cli"
+	nvidiaContainerRuntimeHookSource = "/usr/bin/nvidia-container-toolkit"
+
+	nvidiaContainerToolkitConfigSource = "/etc/nvidia-container-runtime/config.toml"
+	configFilename                     = "config.toml"
+)
+
+var toolkitDirArg string
+var nvidiaDriverRootFlag string
+var nvidiaContainerRuntimeDebugFlag string
+var nvidiaContainerRuntimeLogLevelFlag string
+var nvidiaContainerCLIDebugFlag string
+
+func main() {
+	// Create the top-level CLI
+	c := cli.NewApp()
+	c.Name = "toolkit"
+	c.Usage = "Manage the NVIDIA container toolkit"
+	c.Version = "0.1.0"
+
+	// Create the 'install' subcommand
+	install := cli.Command{}
+	install.Name = "install"
+	install.Usage = "Install the components of the NVIDIA container toolkit"
+	install.ArgsUsage = "<toolkit_directory>"
+	install.Before = parseArgs
+	install.Action = Install
+
+	// Create the 'delete' command
+	delete := cli.Command{}
+	delete.Name = "delete"
+	delete.Usage = "Delete the NVIDIA container toolkit"
+	delete.ArgsUsage = "<toolkit_directory>"
+	delete.Before = parseArgs
+	delete.Action = Delete
+
+	// Register the subcommand with the top-level CLI
+	c.Commands = []*cli.Command{
+		&install,
+		&delete,
+	}
+
+	flags := []cli.Flag{
+		&cli.StringFlag{
+			Name:        "nvidia-driver-root",
+			Value:       DefaultNvidiaDriverRoot,
+			Destination: &nvidiaDriverRootFlag,
+			EnvVars:     []string{"NVIDIA_DRIVER_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime-debug",
+			Usage:       "Specify the location of the debug log file for the NVIDIA Container Runtime",
+			Destination: &nvidiaContainerRuntimeDebugFlag,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_DEBUG"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-runtime-debug-log-level",
+			Destination: &nvidiaContainerRuntimeLogLevelFlag,
+			EnvVars:     []string{"NVIDIA_CONTAINER_RUNTIME_LOG_LEVEL"},
+		},
+		&cli.StringFlag{
+			Name:        "nvidia-container-cli-debug",
+			Usage:       "Specify the location of the debug log file for the NVIDIA Container CLI",
+			Destination: &nvidiaContainerCLIDebugFlag,
+			EnvVars:     []string{"NVIDIA_CONTAINER_CLI_DEBUG"},
+		},
+	}
+
+	// Update the subcommand flags with the common subcommand flags
+	install.Flags = append([]cli.Flag{}, flags...)
+
+	// Run the top-level CLI
+	if err := c.Run(os.Args); err != nil {
+		log.Fatal(fmt.Errorf("error: %v", err))
+	}
+}
+
+// parseArgs parses the command line arguments to the CLI
+func parseArgs(c *cli.Context) error {
+	args := c.Args()
+
+	log.Infof("Parsing arguments: %v", args.Slice())
+	if c.NArg() != 1 {
+		return fmt.Errorf("incorrect number of arguments")
+	}
+	toolkitDirArg = args.Get(0)
+	log.Infof("Successfully parsed arguments")
+
+	return nil
+}
+
+// Delete removes the NVIDIA container toolkit
+func Delete(cli *cli.Context) error {
+	log.Infof("Deleting NVIDIA container toolkit from '%v'", toolkitDirArg)
+	err := os.RemoveAll(toolkitDirArg)
+	if err != nil {
+		return fmt.Errorf("error deleting toolkit directory: %v", err)
+	}
+	return nil
+}
+
+// Install installs the components of the NVIDIA container toolkit.
+// Any existing installation is removed.
+func Install(cli *cli.Context) error {
+	log.Infof("Installing NVIDIA container toolkit to '%v'", toolkitDirArg)
+
+	log.Infof("Removing existing NVIDIA container toolkit installation")
+	err := os.RemoveAll(toolkitDirArg)
+	if err != nil {
+		return fmt.Errorf("error removing toolkit directory: %v", err)
+	}
+
+	toolkitConfigDir := filepath.Join(toolkitDirArg, ".config", "nvidia-container-runtime")
+	toolkitConfigPath := filepath.Join(toolkitConfigDir, configFilename)
+
+	err = createDirectories(toolkitDirArg, toolkitConfigDir)
+	if err != nil {
+		return fmt.Errorf("could not create required directories: %v", err)
+	}
+
+	err = installContainerLibrary(toolkitDirArg)
+	if err != nil {
+		return fmt.Errorf("error installing NVIDIA container library: %v", err)
+	}
+
+	err = installContainerRuntimes(toolkitDirArg, nvidiaDriverRootFlag)
+	if err != nil {
+		return fmt.Errorf("error installing NVIDIA container runtime: %v", err)
+	}
+
+	nvidiaContainerCliExecutable, err := installContainerCLI(toolkitDirArg)
+	if err != nil {
+		return fmt.Errorf("error installing NVIDIA container CLI: %v", err)
+	}
+
+	_, err = installRuntimeHook(toolkitDirArg, toolkitConfigPath)
+	if err != nil {
+		return fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)
+	}
+
+	err = installToolkitConfig(toolkitConfigPath, nvidiaDriverRootFlag, nvidiaContainerCliExecutable)
+	if err != nil {
+		return fmt.Errorf("error installing NVIDIA container toolkit config: %v", err)
+	}
+
+	return nil
+}
+
+// installContainerLibrary locates and installs the libnvidia-container.so.1 library.
+// A predefined set of library candidates are considered, with the first one
+// resulting in success being installed to the toolkit folder. The install process
+// resolves the symlink for the library and copies the versioned library itself.
+func installContainerLibrary(toolkitDir string) error {
+	log.Infof("Installing NVIDIA container library to '%v'", toolkitDir)
+
+	const libName = "libnvidia-container.so.1"
+	libraryPath, err := findLibrary("", libName)
+	if err != nil {
+		return fmt.Errorf("error locating NVIDIA container library: %v", err)
+	}
+
+	installedLibPath, err := installFileToFolder(toolkitDir, libraryPath)
+	if err != nil {
+		return fmt.Errorf("error installing %v to %v: %v", libraryPath, toolkitDir, err)
+	}
+	log.Infof("Installed '%v' to '%v'", libraryPath, installedLibPath)
+
+	if filepath.Base(installedLibPath) == libName {
+		return nil
+	}
+
+	err = installSymlink(toolkitDir, libName, installedLibPath)
+	if err != nil {
+		return fmt.Errorf("error installing symlink for NVIDIA container library: %v", err)
+	}
+
+	return nil
+}
+
+// installToolkitConfig installs the config file for the NVIDIA container toolkit ensuring
+// that the settings are updated to match the desired install and nvidia driver directories.
+func installToolkitConfig(toolkitConfigPath string, nvidiaDriverDir string, nvidiaContainerCliExecutablePath string) error {
+	log.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath)
+
+	config, err := toml.LoadFile(nvidiaContainerToolkitConfigSource)
+	if err != nil {
+		return fmt.Errorf("could not open source config file: %v", err)
+	}
+
+	targetConfig, err := os.Create(toolkitConfigPath)
+	if err != nil {
+		return fmt.Errorf("could not create target config file: %v", err)
+	}
+	defer targetConfig.Close()
+
+	nvidiaContainerCliKey := func(p string) []string {
+		return []string{"nvidia-container-cli", p}
+	}
+
+	// Read the ldconfig path from the config as this may differ per platform
+	// On ubuntu-based systems this ends in `.real`
+	ldconfigPath := fmt.Sprintf("%s", config.GetPath(nvidiaContainerCliKey("ldconfig")))
+
+	// Use the driver run root as the root:
+	driverLdconfigPath := "@" + filepath.Join(nvidiaDriverDir, strings.TrimPrefix(ldconfigPath, "@/"))
+
+	config.SetPath(nvidiaContainerCliKey("root"), nvidiaDriverDir)
+	config.SetPath(nvidiaContainerCliKey("path"), nvidiaContainerCliExecutablePath)
+	config.SetPath(nvidiaContainerCliKey("ldconfig"), driverLdconfigPath)
+
+	// Set the debug options if selected
+	debugOptions := map[string]string{
+		"nvidia-container-runtime.debug":     nvidiaContainerRuntimeDebugFlag,
+		"nvidia-container-runtime.log-level": nvidiaContainerRuntimeLogLevelFlag,
+		"nvidia-container-cli.debug":         nvidiaContainerCLIDebugFlag,
+	}
+	for key, value := range debugOptions {
+		if value == "" {
+			continue
+		}
+		if config.Get(key) != nil {
+			continue
+		}
+		config.Set(key, value)
+	}
+
+	_, err = config.WriteTo(targetConfig)
+	if err != nil {
+		return fmt.Errorf("error writing config: %v", err)
+	}
+	return nil
+}
+
+// installContainerCLI sets up the NVIDIA container CLI executable, copying the executable
+// and implementing the required wrapper
+func installContainerCLI(toolkitDir string) (string, error) {
+	log.Infof("Installing NVIDIA container CLI from '%v'", nvidiaContainerCliSource)
+
+	env := map[string]string{
+		"LD_LIBRARY_PATH": toolkitDir,
+	}
+
+	e := executable{
+		source: nvidiaContainerCliSource,
+		target: executableTarget{
+			dotfileName: "nvidia-container-cli.real",
+			wrapperName: "nvidia-container-cli",
+		},
+		env: env,
+	}
+
+	installedPath, err := e.install(toolkitDir)
+	if err != nil {
+		return "", fmt.Errorf("error installing NVIDIA container CLI: %v", err)
+	}
+	return installedPath, nil
+}
+
+// installRuntimeHook sets up the NVIDIA runtime hook, copying the executable
+// and implementing the required wrapper
+func installRuntimeHook(toolkitDir string, configFilePath string) (string, error) {
+	log.Infof("Installing NVIDIA container runtime hook from '%v'", nvidiaContainerRuntimeHookSource)
+
+	argLines := []string{
+		fmt.Sprintf("-config \"%s\"", configFilePath),
+	}
+
+	e := executable{
+		source: nvidiaContainerRuntimeHookSource,
+		target: executableTarget{
+			dotfileName: "nvidia-container-toolkit.real",
+			wrapperName: "nvidia-container-toolkit",
+		},
+		argLines: argLines,
+	}
+
+	installedPath, err := e.install(toolkitDir)
+	if err != nil {
+		return "", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err)
+	}
+
+	err = installSymlink(toolkitDir, "nvidia-container-runtime-hook", installedPath)
+	if err != nil {
+		return "", fmt.Errorf("error installing symlink to NVIDIA container runtime hook: %v", err)
+	}
+
+	return installedPath, nil
+}
+
+// installSymlink creates a symlink in the toolkitDirectory that points to the specified target.
+// Note: The target is assumed to be local to the toolkit directory
+func installSymlink(toolkitDir string, link string, target string) error {
+	symlinkPath := filepath.Join(toolkitDir, link)
+	targetPath := filepath.Base(target)
+	log.Infof("Creating symlink '%v' -> '%v'", symlinkPath, targetPath)
+
+	err := os.Symlink(targetPath, symlinkPath)
+	if err != nil {
+		return fmt.Errorf("error creating symlink '%v' => '%v': %v", symlinkPath, targetPath, err)
+	}
+	return nil
+}
+
+// installFileToFolder copies a source file to a destination folder.
+// The path of the input file is ignored.
+// e.g. installFileToFolder("/some/path/file.txt", "/output/path")
+// will result in a file "/output/path/file.txt" being generated
+func installFileToFolder(destFolder string, src string) (string, error) {
+	name := filepath.Base(src)
+	return installFileToFolderWithName(destFolder, name, src)
+}
+
+// cp src destFolder/name
+func installFileToFolderWithName(destFolder string, name, src string) (string, error) {
+	dest := filepath.Join(destFolder, name)
+	err := installFile(dest, src)
+	if err != nil {
+		return "", fmt.Errorf("error copying '%v' to '%v': %v", src, dest, err)
+	}
+	return dest, nil
+}
+
+// installFile copies a file from src to dest and maintains
+// file modes
+func installFile(dest string, src string) error {
+	log.Infof("Installing '%v' to '%v'", src, dest)
+
+	source, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("error opening source: %v", err)
+	}
+	defer source.Close()
+
+	destination, err := os.Create(dest)
+	if err != nil {
+		return fmt.Errorf("error creating destination: %v", err)
+	}
+	defer destination.Close()
+
+	_, err = io.Copy(destination, source)
+	if err != nil {
+		return fmt.Errorf("error copying file: %v", err)
+	}
+
+	err = applyModeFromSource(dest, src)
+	if err != nil {
+		return fmt.Errorf("error setting destination file mode: %v", err)
+	}
+	return nil
+}
+
+// applyModeFromSource sets the file mode for a destination file
+// to match that of a specified source file
+func applyModeFromSource(dest string, src string) error {
+	sourceInfo, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("error getting file info for '%v': %v", src, err)
+	}
+	err = os.Chmod(dest, sourceInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("error setting mode for '%v': %v", dest, err)
+	}
+	return nil
+}
+
+// findLibrary searches a set of candidate libraries in the specified root for
+// a given library name
+func findLibrary(root string, libName string) (string, error) {
+	log.Infof("Finding library %v (root=%v)", libName, root)
+
+	candidateDirs := []string{
+		"/usr/lib64",
+		"/usr/lib/x86_64-linux-gnu",
+	}
+
+	for _, d := range candidateDirs {
+		l := filepath.Join(root, d, libName)
+		log.Infof("Checking library candidate '%v'", l)
+
+		libraryCandidate, err := resolveLink(l)
+		if err != nil {
+			log.Infof("Skipping library candidate '%v': %v", l, err)
+			continue
+		}
+
+		return libraryCandidate, nil
+	}
+
+	return "", fmt.Errorf("error locating library '%v'", libName)
+}
+
+// resolveLink finds the target of a symlink or the file itself in the
+// case of a regular file.
+// This is equivalent to running `readlink -f ${l}`
+func resolveLink(l string) (string, error) {
+	resolved, err := filepath.EvalSymlinks(l)
+	if err != nil {
+		return "", fmt.Errorf("error resolving link '%v': %v", l, err)
+	}
+	if l != resolved {
+		log.Infof("Resolved link: '%v' => '%v'", l, resolved)
+	}
+	return resolved, nil
+}
+
+func createDirectories(dir ...string) error {
+	for _, d := range dir {
+		log.Infof("Creating directory '%v'", d)
+		err := os.MkdirAll(d, 0755)
+		if err != nil {
+			return fmt.Errorf("error creating directory: %v", err)
+		}
+	}
+	return nil
+}

vendor/github.com/containerd/containerd/LICENSE

@@ -0,0 +1,191 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        https://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   Copyright The containerd Authors
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       https://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/containerd/containerd/NOTICE

@@ -0,0 +1,16 @@
+Docker
+Copyright 2012-2015 Docker, Inc.
+
+This product includes software developed at Docker, Inc. (https://www.docker.com).
+
+The following is courtesy of our legal counsel:
+
+
+Use and transfer of Docker may be subject to certain restrictions by the
+United States and other governments.
+It is your responsibility to ensure that your use and/or transfer does not
+violate applicable laws.
+
+For more information, please see https://www.bis.doc.gov
+
+See also https://www.apache.org/dev/crypto.html and/or seek legal counsel.

vendor/github.com/containerd/containerd/errdefs/errors.go

@@ -0,0 +1,93 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Package errdefs defines the common errors used throughout containerd
+// packages.
+//
+// Use with errors.Wrap and error.Wrapf to add context to an error.
+//
+// To detect an error class, use the IsXXX functions to tell whether an error
+// is of a certain type.
+//
+// The functions ToGRPC and FromGRPC can be used to map server-side and
+// client-side errors to the correct types.
+package errdefs
+
+import (
+	"context"
+
+	"github.com/pkg/errors"
+)
+
+// Definitions of common error types used throughout containerd. All containerd
+// errors returned by most packages will map into one of these errors classes.
+// Packages should return errors of these types when they want to instruct a
+// client to take a particular action.
+//
+// For the most part, we just try to provide local grpc errors. Most conditions
+// map very well to those defined by grpc.
+var (
+	ErrUnknown            = errors.New("unknown") // used internally to represent a missed mapping.
+	ErrInvalidArgument    = errors.New("invalid argument")
+	ErrNotFound           = errors.New("not found")
+	ErrAlreadyExists      = errors.New("already exists")
+	ErrFailedPrecondition = errors.New("failed precondition")
+	ErrUnavailable        = errors.New("unavailable")
+	ErrNotImplemented     = errors.New("not implemented") // represents not supported and unimplemented
+)
+
+// IsInvalidArgument returns true if the error is due to an invalid argument
+func IsInvalidArgument(err error) bool {
+	return errors.Is(err, ErrInvalidArgument)
+}
+
+// IsNotFound returns true if the error is due to a missing object
+func IsNotFound(err error) bool {
+	return errors.Is(err, ErrNotFound)
+}
+
+// IsAlreadyExists returns true if the error is due to an already existing
+// metadata item
+func IsAlreadyExists(err error) bool {
+	return errors.Is(err, ErrAlreadyExists)
+}
+
+// IsFailedPrecondition returns true if an operation could not proceed to the
+// lack of a particular condition
+func IsFailedPrecondition(err error) bool {
+	return errors.Is(err, ErrFailedPrecondition)
+}
+
+// IsUnavailable returns true if the error is due to a resource being unavailable
+func IsUnavailable(err error) bool {
+	return errors.Is(err, ErrUnavailable)
+}
+
+// IsNotImplemented returns true if the error is due to not being implemented
+func IsNotImplemented(err error) bool {
+	return errors.Is(err, ErrNotImplemented)
+}
+
+// IsCanceled returns true if the error is due to `context.Canceled`.
+func IsCanceled(err error) bool {
+	return errors.Is(err, context.Canceled)
+}
+
+// IsDeadlineExceeded returns true if the error is due to
+// `context.DeadlineExceeded`.
+func IsDeadlineExceeded(err error) bool {
+	return errors.Is(err, context.DeadlineExceeded)
+}

vendor/github.com/containerd/containerd/errdefs/grpc.go

@@ -0,0 +1,147 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package errdefs
+
+import (
+	"context"
+	"strings"
+
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// ToGRPC will attempt to map the backend containerd error into a grpc error,
+// using the original error message as a description.
+//
+// Further information may be extracted from certain errors depending on their
+// type.
+//
+// If the error is unmapped, the original error will be returned to be handled
+// by the regular grpc error handling stack.
+func ToGRPC(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	if isGRPCError(err) {
+		// error has already been mapped to grpc
+		return err
+	}
+
+	switch {
+	case IsInvalidArgument(err):
+		return status.Errorf(codes.InvalidArgument, err.Error())
+	case IsNotFound(err):
+		return status.Errorf(codes.NotFound, err.Error())
+	case IsAlreadyExists(err):
+		return status.Errorf(codes.AlreadyExists, err.Error())
+	case IsFailedPrecondition(err):
+		return status.Errorf(codes.FailedPrecondition, err.Error())
+	case IsUnavailable(err):
+		return status.Errorf(codes.Unavailable, err.Error())
+	case IsNotImplemented(err):
+		return status.Errorf(codes.Unimplemented, err.Error())
+	case IsCanceled(err):
+		return status.Errorf(codes.Canceled, err.Error())
+	case IsDeadlineExceeded(err):
+		return status.Errorf(codes.DeadlineExceeded, err.Error())
+	}
+
+	return err
+}
+
+// ToGRPCf maps the error to grpc error codes, assembling the formatting string
+// and combining it with the target error string.
+//
+// This is equivalent to errors.ToGRPC(errors.Wrapf(err, format, args...))
+func ToGRPCf(err error, format string, args ...interface{}) error {
+	return ToGRPC(errors.Wrapf(err, format, args...))
+}
+
+// FromGRPC returns the underlying error from a grpc service based on the grpc error code
+func FromGRPC(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	var cls error // divide these into error classes, becomes the cause
+
+	switch code(err) {
+	case codes.InvalidArgument:
+		cls = ErrInvalidArgument
+	case codes.AlreadyExists:
+		cls = ErrAlreadyExists
+	case codes.NotFound:
+		cls = ErrNotFound
+	case codes.Unavailable:
+		cls = ErrUnavailable
+	case codes.FailedPrecondition:
+		cls = ErrFailedPrecondition
+	case codes.Unimplemented:
+		cls = ErrNotImplemented
+	case codes.Canceled:
+		cls = context.Canceled
+	case codes.DeadlineExceeded:
+		cls = context.DeadlineExceeded
+	default:
+		cls = ErrUnknown
+	}
+
+	msg := rebaseMessage(cls, err)
+	if msg != "" {
+		err = errors.Wrap(cls, msg)
+	} else {
+		err = errors.WithStack(cls)
+	}
+
+	return err
+}
+
+// rebaseMessage removes the repeats for an error at the end of an error
+// string. This will happen when taking an error over grpc then remapping it.
+//
+// Effectively, we just remove the string of cls from the end of err if it
+// appears there.
+func rebaseMessage(cls error, err error) string {
+	desc := errDesc(err)
+	clss := cls.Error()
+	if desc == clss {
+		return ""
+	}
+
+	return strings.TrimSuffix(desc, ": "+clss)
+}
+
+func isGRPCError(err error) bool {
+	_, ok := status.FromError(err)
+	return ok
+}
+
+func code(err error) codes.Code {
+	if s, ok := status.FromError(err); ok {
+		return s.Code()
+	}
+	return codes.Unknown
+}
+
+func errDesc(err error) string {
+	if s, ok := status.FromError(err); ok {
+		return s.Message()
+	}
+	return err.Error()
+}

vendor/github.com/containerd/containerd/events/events.go

@@ -0,0 +1,81 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package events
+
+import (
+	"context"
+	"time"
+
+	"github.com/containerd/typeurl"
+	"github.com/gogo/protobuf/types"
+)
+
+// Envelope provides the packaging for an event.
+type Envelope struct {
+	Timestamp time.Time
+	Namespace string
+	Topic     string
+	Event     *types.Any
+}
+
+// Field returns the value for the given fieldpath as a string, if defined.
+// If the value is not defined, the second value will be false.
+func (e *Envelope) Field(fieldpath []string) (string, bool) {
+	if len(fieldpath) == 0 {
+		return "", false
+	}
+
+	switch fieldpath[0] {
+	// unhandled: timestamp
+	case "namespace":
+		return e.Namespace, len(e.Namespace) > 0
+	case "topic":
+		return e.Topic, len(e.Topic) > 0
+	case "event":
+		decoded, err := typeurl.UnmarshalAny(e.Event)
+		if err != nil {
+			return "", false
+		}
+
+		adaptor, ok := decoded.(interface {
+			Field([]string) (string, bool)
+		})
+		if !ok {
+			return "", false
+		}
+		return adaptor.Field(fieldpath[1:])
+	}
+	return "", false
+}
+
+// Event is a generic interface for any type of event
+type Event interface{}
+
+// Publisher posts the event.
+type Publisher interface {
+	Publish(ctx context.Context, topic string, event Event) error
+}
+
+// Forwarder forwards an event to the underlying event bus
+type Forwarder interface {
+	Forward(ctx context.Context, envelope *Envelope) error
+}
+
+// Subscriber allows callers to subscribe to events
+type Subscriber interface {
+	Subscribe(ctx context.Context, filters ...string) (ch <-chan *Envelope, errs <-chan error)
+}

vendor/github.com/containerd/containerd/events/exchange/exchange.go

@@ -0,0 +1,251 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package exchange
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/events"
+	"github.com/containerd/containerd/filters"
+	"github.com/containerd/containerd/identifiers"
+	"github.com/containerd/containerd/log"
+	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/typeurl"
+	goevents "github.com/docker/go-events"
+	"github.com/gogo/protobuf/types"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Exchange broadcasts events
+type Exchange struct {
+	broadcaster *goevents.Broadcaster
+}
+
+// NewExchange returns a new event Exchange
+func NewExchange() *Exchange {
+	return &Exchange{
+		broadcaster: goevents.NewBroadcaster(),
+	}
+}
+
+var _ events.Publisher = &Exchange{}
+var _ events.Forwarder = &Exchange{}
+var _ events.Subscriber = &Exchange{}
+
+// Forward accepts an envelope to be directly distributed on the exchange.
+//
+// This is useful when an event is forwarded on behalf of another namespace or
+// when the event is propagated on behalf of another publisher.
+func (e *Exchange) Forward(ctx context.Context, envelope *events.Envelope) (err error) {
+	if err := validateEnvelope(envelope); err != nil {
+		return err
+	}
+
+	defer func() {
+		logger := log.G(ctx).WithFields(logrus.Fields{
+			"topic": envelope.Topic,
+			"ns":    envelope.Namespace,
+			"type":  envelope.Event.TypeUrl,
+		})
+
+		if err != nil {
+			logger.WithError(err).Error("error forwarding event")
+		} else {
+			logger.Debug("event forwarded")
+		}
+	}()
+
+	return e.broadcaster.Write(envelope)
+}
+
+// Publish packages and sends an event. The caller will be considered the
+// initial publisher of the event. This means the timestamp will be calculated
+// at this point and this method may read from the calling context.
+func (e *Exchange) Publish(ctx context.Context, topic string, event events.Event) (err error) {
+	var (
+		namespace string
+		encoded   *types.Any
+		envelope  events.Envelope
+	)
+
+	namespace, err = namespaces.NamespaceRequired(ctx)
+	if err != nil {
+		return errors.Wrapf(err, "failed publishing event")
+	}
+	if err := validateTopic(topic); err != nil {
+		return errors.Wrapf(err, "envelope topic %q", topic)
+	}
+
+	encoded, err = typeurl.MarshalAny(event)
+	if err != nil {
+		return err
+	}
+
+	envelope.Timestamp = time.Now().UTC()
+	envelope.Namespace = namespace
+	envelope.Topic = topic
+	envelope.Event = encoded
+
+	defer func() {
+		logger := log.G(ctx).WithFields(logrus.Fields{
+			"topic": envelope.Topic,
+			"ns":    envelope.Namespace,
+			"type":  envelope.Event.TypeUrl,
+		})
+
+		if err != nil {
+			logger.WithError(err).Error("error publishing event")
+		} else {
+			logger.Debug("event published")
+		}
+	}()
+
+	return e.broadcaster.Write(&envelope)
+}
+
+// Subscribe to events on the exchange. Events are sent through the returned
+// channel ch. If an error is encountered, it will be sent on channel errs and
+// errs will be closed. To end the subscription, cancel the provided context.
+//
+// Zero or more filters may be provided as strings. Only events that match
+// *any* of the provided filters will be sent on the channel. The filters use
+// the standard containerd filters package syntax.
+func (e *Exchange) Subscribe(ctx context.Context, fs ...string) (ch <-chan *events.Envelope, errs <-chan error) {
+	var (
+		evch                  = make(chan *events.Envelope)
+		errq                  = make(chan error, 1)
+		channel               = goevents.NewChannel(0)
+		queue                 = goevents.NewQueue(channel)
+		dst     goevents.Sink = queue
+	)
+
+	closeAll := func() {
+		channel.Close()
+		queue.Close()
+		e.broadcaster.Remove(dst)
+		close(errq)
+	}
+
+	ch = evch
+	errs = errq
+
+	if len(fs) > 0 {
+		filter, err := filters.ParseAll(fs...)
+		if err != nil {
+			errq <- errors.Wrapf(err, "failed parsing subscription filters")
+			closeAll()
+			return
+		}
+
+		dst = goevents.NewFilter(queue, goevents.MatcherFunc(func(gev goevents.Event) bool {
+			return filter.Match(adapt(gev))
+		}))
+	}
+
+	e.broadcaster.Add(dst)
+
+	go func() {
+		defer closeAll()
+
+		var err error
+	loop:
+		for {
+			select {
+			case ev := <-channel.C:
+				env, ok := ev.(*events.Envelope)
+				if !ok {
+					// TODO(stevvooe): For the most part, we are well protected
+					// from this condition. Both Forward and Publish protect
+					// from this.
+					err = errors.Errorf("invalid envelope encountered %#v; please file a bug", ev)
+					break
+				}
+
+				select {
+				case evch <- env:
+				case <-ctx.Done():
+					break loop
+				}
+			case <-ctx.Done():
+				break loop
+			}
+		}
+
+		if err == nil {
+			if cerr := ctx.Err(); cerr != context.Canceled {
+				err = cerr
+			}
+		}
+
+		errq <- err
+	}()
+
+	return
+}
+
+func validateTopic(topic string) error {
+	if topic == "" {
+		return errors.Wrap(errdefs.ErrInvalidArgument, "must not be empty")
+	}
+
+	if topic[0] != '/' {
+		return errors.Wrapf(errdefs.ErrInvalidArgument, "must start with '/'")
+	}
+
+	if len(topic) == 1 {
+		return errors.Wrapf(errdefs.ErrInvalidArgument, "must have at least one component")
+	}
+
+	components := strings.Split(topic[1:], "/")
+	for _, component := range components {
+		if err := identifiers.Validate(component); err != nil {
+			return errors.Wrapf(err, "failed validation on component %q", component)
+		}
+	}
+
+	return nil
+}
+
+func validateEnvelope(envelope *events.Envelope) error {
+	if err := identifiers.Validate(envelope.Namespace); err != nil {
+		return errors.Wrapf(err, "event envelope has invalid namespace")
+	}
+
+	if err := validateTopic(envelope.Topic); err != nil {
+		return errors.Wrapf(err, "envelope topic %q", envelope.Topic)
+	}
+
+	if envelope.Timestamp.IsZero() {
+		return errors.Wrapf(errdefs.ErrInvalidArgument, "timestamp must be set on forwarded event")
+	}
+
+	return nil
+}
+
+func adapt(ev interface{}) filters.Adaptor {
+	if adaptor, ok := ev.(filters.Adaptor); ok {
+		return adaptor
+	}
+
+	return filters.AdapterFunc(func(fieldpath []string) (string, bool) {
+		return "", false
+	})
+}

vendor/github.com/containerd/containerd/filters/adaptor.go

@@ -0,0 +1,33 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package filters
+
+// Adaptor specifies the mapping of fieldpaths to a type. For the given field
+// path, the value and whether it is present should be returned. The mapping of
+// the fieldpath to a field is deferred to the adaptor implementation, but
+// should generally follow protobuf field path/mask semantics.
+type Adaptor interface {
+	Field(fieldpath []string) (value string, present bool)
+}
+
+// AdapterFunc allows implementation specific matching of fieldpaths
+type AdapterFunc func(fieldpath []string) (string, bool)
+
+// Field returns the field name and true if it exists
+func (fn AdapterFunc) Field(fieldpath []string) (string, bool) {
+	return fn(fieldpath)
+}

vendor/github.com/containerd/containerd/filters/filter.go

@@ -0,0 +1,179 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Package filters defines a syntax and parser that can be used for the
+// filtration of items across the containerd API. The core is built on the
+// concept of protobuf field paths, with quoting.  Several operators allow the
+// user to flexibly select items based on field presence, equality, inequality
+// and regular expressions. Flexible adaptors support working with any type.
+//
+// The syntax is fairly familiar, if you've used container ecosystem
+// projects.  At the core, we base it on the concept of protobuf field
+// paths, augmenting with the ability to quote portions of the field path
+// to match arbitrary labels. These "selectors" come in the following
+// syntax:
+//
+// ```
+// <fieldpath>[<operator><value>]
+// ```
+//
+// A basic example is as follows:
+//
+// ```
+// name==foo
+// ```
+//
+// This would match all objects that have a field `name` with the value
+// `foo`. If we only want to test if the field is present, we can omit the
+// operator. This is most useful for matching labels in containerd. The
+// following will match objects that have the field "labels" and have the
+// label "foo" defined:
+//
+// ```
+// labels.foo
+// ```
+//
+// We also allow for quoting of parts of the field path to allow matching
+// of arbitrary items:
+//
+// ```
+// labels."very complex label"==something
+// ```
+//
+// We also define `!=` and `~=` as operators. The `!=` will match all
+// objects that don't match the value for a field and `~=` will compile the
+// target value as a regular expression and match the field value against that.
+//
+// Selectors can be combined using a comma, such that the resulting
+// selector will require all selectors are matched for the object to match.
+// The following example will match objects that are named `foo` and have
+// the label `bar`:
+//
+// ```
+// name==foo,labels.bar
+// ```
+//
+package filters
+
+import (
+	"regexp"
+
+	"github.com/containerd/containerd/log"
+)
+
+// Filter matches specific resources based the provided filter
+type Filter interface {
+	Match(adaptor Adaptor) bool
+}
+
+// FilterFunc is a function that handles matching with an adaptor
+type FilterFunc func(Adaptor) bool
+
+// Match matches the FilterFunc returning true if the object matches the filter
+func (fn FilterFunc) Match(adaptor Adaptor) bool {
+	return fn(adaptor)
+}
+
+// Always is a filter that always returns true for any type of object
+var Always FilterFunc = func(adaptor Adaptor) bool {
+	return true
+}
+
+// Any allows multiple filters to be matched against the object
+type Any []Filter
+
+// Match returns true if any of the provided filters are true
+func (m Any) Match(adaptor Adaptor) bool {
+	for _, m := range m {
+		if m.Match(adaptor) {
+			return true
+		}
+	}
+
+	return false
+}
+
+// All allows multiple filters to be matched against the object
+type All []Filter
+
+// Match only returns true if all filters match the object
+func (m All) Match(adaptor Adaptor) bool {
+	for _, m := range m {
+		if !m.Match(adaptor) {
+			return false
+		}
+	}
+
+	return true
+}
+
+type operator int
+
+const (
+	operatorPresent = iota
+	operatorEqual
+	operatorNotEqual
+	operatorMatches
+)
+
+func (op operator) String() string {
+	switch op {
+	case operatorPresent:
+		return "?"
+	case operatorEqual:
+		return "=="
+	case operatorNotEqual:
+		return "!="
+	case operatorMatches:
+		return "~="
+	}
+
+	return "unknown"
+}
+
+type selector struct {
+	fieldpath []string
+	operator  operator
+	value     string
+	re        *regexp.Regexp
+}
+
+func (m selector) Match(adaptor Adaptor) bool {
+	value, present := adaptor.Field(m.fieldpath)
+
+	switch m.operator {
+	case operatorPresent:
+		return present
+	case operatorEqual:
+		return present && value == m.value
+	case operatorNotEqual:
+		return value != m.value
+	case operatorMatches:
+		if m.re == nil {
+			r, err := regexp.Compile(m.value)
+			if err != nil {
+				log.L.Errorf("error compiling regexp %q", m.value)
+				return false
+			}
+
+			m.re = r
+		}
+
+		return m.re.MatchString(value)
+	default:
+		return false
+	}
+}

vendor/github.com/containerd/containerd/filters/parser.go

@@ -0,0 +1,292 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package filters
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/containerd/containerd/errdefs"
+	"github.com/pkg/errors"
+)
+
+/*
+Parse the strings into a filter that may be used with an adaptor.
+
+The filter is made up of zero or more selectors.
+
+The format is a comma separated list of expressions, in the form of
+`<fieldpath><op><value>`, known as selectors. All selectors must match the
+target object for the filter to be true.
+
+We define the operators "==" for equality, "!=" for not equal and "~=" for a
+regular expression. If the operator and value are not present, the matcher will
+test for the presence of a value, as defined by the target object.
+
+The formal grammar is as follows:
+
+selectors := selector ("," selector)*
+selector  := fieldpath (operator value)
+fieldpath := field ('.' field)*
+field     := quoted | [A-Za-z] [A-Za-z0-9_]+
+operator  := "==" | "!=" | "~="
+value     := quoted | [^\s,]+
+quoted    := <go string syntax>
+
+*/
+func Parse(s string) (Filter, error) {
+	// special case empty to match all
+	if s == "" {
+		return Always, nil
+	}
+
+	p := parser{input: s}
+	return p.parse()
+}
+
+// ParseAll parses each filter in ss and returns a filter that will return true
+// if any filter matches the expression.
+//
+// If no filters are provided, the filter will match anything.
+func ParseAll(ss ...string) (Filter, error) {
+	if len(ss) == 0 {
+		return Always, nil
+	}
+
+	var fs []Filter
+	for _, s := range ss {
+		f, err := Parse(s)
+		if err != nil {
+			return nil, errors.Wrap(errdefs.ErrInvalidArgument, err.Error())
+		}
+
+		fs = append(fs, f)
+	}
+
+	return Any(fs), nil
+}
+
+type parser struct {
+	input   string
+	scanner scanner
+}
+
+func (p *parser) parse() (Filter, error) {
+	p.scanner.init(p.input)
+
+	ss, err := p.selectors()
+	if err != nil {
+		return nil, errors.Wrap(err, "filters")
+	}
+
+	return ss, nil
+}
+
+func (p *parser) selectors() (Filter, error) {
+	s, err := p.selector()
+	if err != nil {
+		return nil, err
+	}
+
+	ss := All{s}
+
+loop:
+	for {
+		tok := p.scanner.peek()
+		switch tok {
+		case ',':
+			pos, tok, _ := p.scanner.scan()
+			if tok != tokenSeparator {
+				return nil, p.mkerr(pos, "expected a separator")
+			}
+
+			s, err := p.selector()
+			if err != nil {
+				return nil, err
+			}
+
+			ss = append(ss, s)
+		case tokenEOF:
+			break loop
+		default:
+			return nil, p.mkerr(p.scanner.ppos, "unexpected input: %v", string(tok))
+		}
+	}
+
+	return ss, nil
+}
+
+func (p *parser) selector() (selector, error) {
+	fieldpath, err := p.fieldpath()
+	if err != nil {
+		return selector{}, err
+	}
+
+	switch p.scanner.peek() {
+	case ',', tokenSeparator, tokenEOF:
+		return selector{
+			fieldpath: fieldpath,
+			operator:  operatorPresent,
+		}, nil
+	}
+
+	op, err := p.operator()
+	if err != nil {
+		return selector{}, err
+	}
+
+	var allowAltQuotes bool
+	if op == operatorMatches {
+		allowAltQuotes = true
+	}
+
+	value, err := p.value(allowAltQuotes)
+	if err != nil {
+		if err == io.EOF {
+			return selector{}, io.ErrUnexpectedEOF
+		}
+		return selector{}, err
+	}
+
+	return selector{
+		fieldpath: fieldpath,
+		value:     value,
+		operator:  op,
+	}, nil
+}
+
+func (p *parser) fieldpath() ([]string, error) {
+	f, err := p.field()
+	if err != nil {
+		return nil, err
+	}
+
+	fs := []string{f}
+loop:
+	for {
+		tok := p.scanner.peek() // lookahead to consume field separator
+
+		switch tok {
+		case '.':
+			pos, tok, _ := p.scanner.scan() // consume separator
+			if tok != tokenSeparator {
+				return nil, p.mkerr(pos, "expected a field separator (`.`)")
+			}
+
+			f, err := p.field()
+			if err != nil {
+				return nil, err
+			}
+
+			fs = append(fs, f)
+		default:
+			// let the layer above handle the other bad cases.
+			break loop
+		}
+	}
+
+	return fs, nil
+}
+
+func (p *parser) field() (string, error) {
+	pos, tok, s := p.scanner.scan()
+	switch tok {
+	case tokenField:
+		return s, nil
+	case tokenQuoted:
+		return p.unquote(pos, s, false)
+	case tokenIllegal:
+		return "", p.mkerr(pos, p.scanner.err)
+	}
+
+	return "", p.mkerr(pos, "expected field or quoted")
+}
+
+func (p *parser) operator() (operator, error) {
+	pos, tok, s := p.scanner.scan()
+	switch tok {
+	case tokenOperator:
+		switch s {
+		case "==":
+			return operatorEqual, nil
+		case "!=":
+			return operatorNotEqual, nil
+		case "~=":
+			return operatorMatches, nil
+		default:
+			return 0, p.mkerr(pos, "unsupported operator %q", s)
+		}
+	case tokenIllegal:
+		return 0, p.mkerr(pos, p.scanner.err)
+	}
+
+	return 0, p.mkerr(pos, `expected an operator ("=="|"!="|"~=")`)
+}
+
+func (p *parser) value(allowAltQuotes bool) (string, error) {
+	pos, tok, s := p.scanner.scan()
+
+	switch tok {
+	case tokenValue, tokenField:
+		return s, nil
+	case tokenQuoted:
+		return p.unquote(pos, s, allowAltQuotes)
+	case tokenIllegal:
+		return "", p.mkerr(pos, p.scanner.err)
+	}
+
+	return "", p.mkerr(pos, "expected value or quoted")
+}
+
+func (p *parser) unquote(pos int, s string, allowAlts bool) (string, error) {
+	if !allowAlts && s[0] != '\'' && s[0] != '"' {
+		return "", p.mkerr(pos, "invalid quote encountered")
+	}
+
+	uq, err := unquote(s)
+	if err != nil {
+		return "", p.mkerr(pos, "unquoting failed: %v", err)
+	}
+
+	return uq, nil
+}
+
+type parseError struct {
+	input string
+	pos   int
+	msg   string
+}
+
+func (pe parseError) Error() string {
+	if pe.pos < len(pe.input) {
+		before := pe.input[:pe.pos]
+		location := pe.input[pe.pos : pe.pos+1] // need to handle end
+		after := pe.input[pe.pos+1:]
+
+		return fmt.Sprintf("[%s >|%s|< %s]: %v", before, location, after, pe.msg)
+	}
+
+	return fmt.Sprintf("[%s]: %v", pe.input, pe.msg)
+}
+
+func (p *parser) mkerr(pos int, format string, args ...interface{}) error {
+	return errors.Wrap(parseError{
+		input: p.input,
+		pos:   pos,
+		msg:   fmt.Sprintf(format, args...),
+	}, "parse error")
+}

vendor/github.com/containerd/containerd/filters/quote.go

@@ -0,0 +1,253 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package filters
+
+import (
+	"unicode/utf8"
+
+	"github.com/pkg/errors"
+)
+
+// NOTE(stevvooe): Most of this code in this file is copied from the stdlib
+// strconv package and modified to be able to handle quoting with `/` and `|`
+// as delimiters.  The copyright is held by the Go authors.
+
+var errQuoteSyntax = errors.New("quote syntax error")
+
+// UnquoteChar decodes the first character or byte in the escaped string
+// or character literal represented by the string s.
+// It returns four values:
+//
+//	1) value, the decoded Unicode code point or byte value;
+//	2) multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation;
+//	3) tail, the remainder of the string after the character; and
+//	4) an error that will be nil if the character is syntactically valid.
+//
+// The second argument, quote, specifies the type of literal being parsed
+// and therefore which escaped quote character is permitted.
+// If set to a single quote, it permits the sequence \' and disallows unescaped '.
+// If set to a double quote, it permits \" and disallows unescaped ".
+// If set to zero, it does not permit either escape and allows both quote characters to appear unescaped.
+//
+// This is from Go strconv package, modified to support `|` and `/` as double
+// quotes for use with regular expressions.
+func unquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) {
+	// easy cases
+	switch c := s[0]; {
+	case c == quote && (quote == '\'' || quote == '"' || quote == '/' || quote == '|'):
+		err = errQuoteSyntax
+		return
+	case c >= utf8.RuneSelf:
+		r, size := utf8.DecodeRuneInString(s)
+		return r, true, s[size:], nil
+	case c != '\\':
+		return rune(s[0]), false, s[1:], nil
+	}
+
+	// hard case: c is backslash
+	if len(s) <= 1 {
+		err = errQuoteSyntax
+		return
+	}
+	c := s[1]
+	s = s[2:]
+
+	switch c {
+	case 'a':
+		value = '\a'
+	case 'b':
+		value = '\b'
+	case 'f':
+		value = '\f'
+	case 'n':
+		value = '\n'
+	case 'r':
+		value = '\r'
+	case 't':
+		value = '\t'
+	case 'v':
+		value = '\v'
+	case 'x', 'u', 'U':
+		n := 0
+		switch c {
+		case 'x':
+			n = 2
+		case 'u':
+			n = 4
+		case 'U':
+			n = 8
+		}
+		var v rune
+		if len(s) < n {
+			err = errQuoteSyntax
+			return
+		}
+		for j := 0; j < n; j++ {
+			x, ok := unhex(s[j])
+			if !ok {
+				err = errQuoteSyntax
+				return
+			}
+			v = v<<4 | x
+		}
+		s = s[n:]
+		if c == 'x' {
+			// single-byte string, possibly not UTF-8
+			value = v
+			break
+		}
+		if v > utf8.MaxRune {
+			err = errQuoteSyntax
+			return
+		}
+		value = v
+		multibyte = true
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		v := rune(c) - '0'
+		if len(s) < 2 {
+			err = errQuoteSyntax
+			return
+		}
+		for j := 0; j < 2; j++ { // one digit already; two more
+			x := rune(s[j]) - '0'
+			if x < 0 || x > 7 {
+				err = errQuoteSyntax
+				return
+			}
+			v = (v << 3) | x
+		}
+		s = s[2:]
+		if v > 255 {
+			err = errQuoteSyntax
+			return
+		}
+		value = v
+	case '\\':
+		value = '\\'
+	case '\'', '"', '|', '/':
+		if c != quote {
+			err = errQuoteSyntax
+			return
+		}
+		value = rune(c)
+	default:
+		err = errQuoteSyntax
+		return
+	}
+	tail = s
+	return
+}
+
+// unquote interprets s as a single-quoted, double-quoted,
+// or backquoted Go string literal, returning the string value
+// that s quotes.  (If s is single-quoted, it would be a Go
+// character literal; Unquote returns the corresponding
+// one-character string.)
+//
+// This is modified from the standard library to support `|` and `/` as quote
+// characters for use with regular expressions.
+func unquote(s string) (string, error) {
+	n := len(s)
+	if n < 2 {
+		return "", errQuoteSyntax
+	}
+	quote := s[0]
+	if quote != s[n-1] {
+		return "", errQuoteSyntax
+	}
+	s = s[1 : n-1]
+
+	if quote == '`' {
+		if contains(s, '`') {
+			return "", errQuoteSyntax
+		}
+		if contains(s, '\r') {
+			// -1 because we know there is at least one \r to remove.
+			buf := make([]byte, 0, len(s)-1)
+			for i := 0; i < len(s); i++ {
+				if s[i] != '\r' {
+					buf = append(buf, s[i])
+				}
+			}
+			return string(buf), nil
+		}
+		return s, nil
+	}
+	if quote != '"' && quote != '\'' && quote != '|' && quote != '/' {
+		return "", errQuoteSyntax
+	}
+	if contains(s, '\n') {
+		return "", errQuoteSyntax
+	}
+
+	// Is it trivial?  Avoid allocation.
+	if !contains(s, '\\') && !contains(s, quote) {
+		switch quote {
+		case '"', '/', '|': // pipe and slash are treated like double quote
+			return s, nil
+		case '\'':
+			r, size := utf8.DecodeRuneInString(s)
+			if size == len(s) && (r != utf8.RuneError || size != 1) {
+				return s, nil
+			}
+		}
+	}
+
+	var runeTmp [utf8.UTFMax]byte
+	buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations.
+	for len(s) > 0 {
+		c, multibyte, ss, err := unquoteChar(s, quote)
+		if err != nil {
+			return "", err
+		}
+		s = ss
+		if c < utf8.RuneSelf || !multibyte {
+			buf = append(buf, byte(c))
+		} else {
+			n := utf8.EncodeRune(runeTmp[:], c)
+			buf = append(buf, runeTmp[:n]...)
+		}
+		if quote == '\'' && len(s) != 0 {
+			// single-quoted must be single character
+			return "", errQuoteSyntax
+		}
+	}
+	return string(buf), nil
+}
+
+// contains reports whether the string contains the byte c.
+func contains(s string, c byte) bool {
+	for i := 0; i < len(s); i++ {
+		if s[i] == c {
+			return true
+		}
+	}
+	return false
+}
+
+func unhex(b byte) (v rune, ok bool) {
+	c := rune(b)
+	switch {
+	case '0' <= c && c <= '9':
+		return c - '0', true
+	case 'a' <= c && c <= 'f':
+		return c - 'a' + 10, true
+	case 'A' <= c && c <= 'F':
+		return c - 'A' + 10, true
+	}
+	return
+}

vendor/github.com/containerd/containerd/filters/scanner.go

@@ -0,0 +1,297 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package filters
+
+import (
+	"unicode"
+	"unicode/utf8"
+)
+
+const (
+	tokenEOF = -(iota + 1)
+	tokenQuoted
+	tokenValue
+	tokenField
+	tokenSeparator
+	tokenOperator
+	tokenIllegal
+)
+
+type token rune
+
+func (t token) String() string {
+	switch t {
+	case tokenEOF:
+		return "EOF"
+	case tokenQuoted:
+		return "Quoted"
+	case tokenValue:
+		return "Value"
+	case tokenField:
+		return "Field"
+	case tokenSeparator:
+		return "Separator"
+	case tokenOperator:
+		return "Operator"
+	case tokenIllegal:
+		return "Illegal"
+	}
+
+	return string(t)
+}
+
+func (t token) GoString() string {
+	return "token" + t.String()
+}
+
+type scanner struct {
+	input string
+	pos   int
+	ppos  int // bounds the current rune in the string
+	value bool
+	err   string
+}
+
+func (s *scanner) init(input string) {
+	s.input = input
+	s.pos = 0
+	s.ppos = 0
+}
+
+func (s *scanner) next() rune {
+	if s.pos >= len(s.input) {
+		return tokenEOF
+	}
+	s.pos = s.ppos
+
+	r, w := utf8.DecodeRuneInString(s.input[s.ppos:])
+	s.ppos += w
+	if r == utf8.RuneError {
+		if w > 0 {
+			s.error("rune error")
+			return tokenIllegal
+		}
+		return tokenEOF
+	}
+
+	if r == 0 {
+		s.error("unexpected null")
+		return tokenIllegal
+	}
+
+	return r
+}
+
+func (s *scanner) peek() rune {
+	pos := s.pos
+	ppos := s.ppos
+	ch := s.next()
+	s.pos = pos
+	s.ppos = ppos
+	return ch
+}
+
+func (s *scanner) scan() (nextp int, tk token, text string) {
+	var (
+		ch  = s.next()
+		pos = s.pos
+	)
+
+chomp:
+	switch {
+	case ch == tokenEOF:
+	case ch == tokenIllegal:
+	case isQuoteRune(ch):
+		if !s.scanQuoted(ch) {
+			return pos, tokenIllegal, s.input[pos:s.ppos]
+		}
+		return pos, tokenQuoted, s.input[pos:s.ppos]
+	case isSeparatorRune(ch):
+		s.value = false
+		return pos, tokenSeparator, s.input[pos:s.ppos]
+	case isOperatorRune(ch):
+		s.scanOperator()
+		s.value = true
+		return pos, tokenOperator, s.input[pos:s.ppos]
+	case unicode.IsSpace(ch):
+		// chomp
+		ch = s.next()
+		pos = s.pos
+		goto chomp
+	case s.value:
+		s.scanValue()
+		s.value = false
+		return pos, tokenValue, s.input[pos:s.ppos]
+	case isFieldRune(ch):
+		s.scanField()
+		return pos, tokenField, s.input[pos:s.ppos]
+	}
+
+	return s.pos, token(ch), ""
+}
+
+func (s *scanner) scanField() {
+	for {
+		ch := s.peek()
+		if !isFieldRune(ch) {
+			break
+		}
+		s.next()
+	}
+}
+
+func (s *scanner) scanOperator() {
+	for {
+		ch := s.peek()
+		switch ch {
+		case '=', '!', '~':
+			s.next()
+		default:
+			return
+		}
+	}
+}
+
+func (s *scanner) scanValue() {
+	for {
+		ch := s.peek()
+		if !isValueRune(ch) {
+			break
+		}
+		s.next()
+	}
+}
+
+func (s *scanner) scanQuoted(quote rune) bool {
+	var illegal bool
+	ch := s.next() // read character after quote
+	for ch != quote {
+		if ch == '\n' || ch < 0 {
+			s.error("quoted literal not terminated")
+			return false
+		}
+		if ch == '\\' {
+			var legal bool
+			ch, legal = s.scanEscape(quote)
+			if !legal {
+				illegal = true
+			}
+		} else {
+			ch = s.next()
+		}
+	}
+	return !illegal
+}
+
+func (s *scanner) scanEscape(quote rune) (ch rune, legal bool) {
+	ch = s.next() // read character after '/'
+	switch ch {
+	case 'a', 'b', 'f', 'n', 'r', 't', 'v', '\\', quote:
+		// nothing to do
+		ch = s.next()
+		legal = true
+	case '0', '1', '2', '3', '4', '5', '6', '7':
+		ch, legal = s.scanDigits(ch, 8, 3)
+	case 'x':
+		ch, legal = s.scanDigits(s.next(), 16, 2)
+	case 'u':
+		ch, legal = s.scanDigits(s.next(), 16, 4)
+	case 'U':
+		ch, legal = s.scanDigits(s.next(), 16, 8)
+	default:
+		s.error("illegal escape sequence")
+	}
+	return
+}
+
+func (s *scanner) scanDigits(ch rune, base, n int) (rune, bool) {
+	for n > 0 && digitVal(ch) < base {
+		ch = s.next()
+		n--
+	}
+	if n > 0 {
+		s.error("illegal numeric escape sequence")
+		return ch, false
+	}
+	return ch, true
+}
+
+func (s *scanner) error(msg string) {
+	if s.err == "" {
+		s.err = msg
+	}
+}
+
+func digitVal(ch rune) int {
+	switch {
+	case '0' <= ch && ch <= '9':
+		return int(ch - '0')
+	case 'a' <= ch && ch <= 'f':
+		return int(ch - 'a' + 10)
+	case 'A' <= ch && ch <= 'F':
+		return int(ch - 'A' + 10)
+	}
+	return 16 // larger than any legal digit val
+}
+
+func isFieldRune(r rune) bool {
+	return (r == '_' || isAlphaRune(r) || isDigitRune(r))
+}
+
+func isAlphaRune(r rune) bool {
+	return r >= 'A' && r <= 'Z' || r >= 'a' && r <= 'z'
+}
+
+func isDigitRune(r rune) bool {
+	return r >= '0' && r <= '9'
+}
+
+func isOperatorRune(r rune) bool {
+	switch r {
+	case '=', '!', '~':
+		return true
+	}
+
+	return false
+}
+
+func isQuoteRune(r rune) bool {
+	switch r {
+	case '/', '|', '"': // maybe add single quoting?
+		return true
+	}
+
+	return false
+}
+
+func isSeparatorRune(r rune) bool {
+	switch r {
+	case ',', '.':
+		return true
+	}
+
+	return false
+}
+
+func isValueRune(r rune) bool {
+	return r != ',' && !unicode.IsSpace(r) &&
+		(unicode.IsLetter(r) ||
+			unicode.IsDigit(r) ||
+			unicode.IsNumber(r) ||
+			unicode.IsGraphic(r) ||
+			unicode.IsPunct(r))
+}

vendor/github.com/containerd/containerd/identifiers/validate.go

@@ -0,0 +1,73 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+// Package identifiers provides common validation for identifiers and keys
+// across containerd.
+//
+// Identifiers in containerd must be a alphanumeric, allowing limited
+// underscores, dashes and dots.
+//
+// While the character set may be expanded in the future, identifiers
+// are guaranteed to be safely used as filesystem path components.
+package identifiers
+
+import (
+	"regexp"
+
+	"github.com/containerd/containerd/errdefs"
+	"github.com/pkg/errors"
+)
+
+const (
+	maxLength  = 76
+	alphanum   = `[A-Za-z0-9]+`
+	separators = `[._-]`
+)
+
+var (
+	// identifierRe defines the pattern for valid identifiers.
+	identifierRe = regexp.MustCompile(reAnchor(alphanum + reGroup(separators+reGroup(alphanum)) + "*"))
+)
+
+// Validate returns nil if the string s is a valid identifier.
+//
+// identifiers are similar to the domain name rules according to RFC 1035, section 2.3.1. However
+// rules in this package are relaxed to allow numerals to follow period (".") and mixed case is
+// allowed.
+//
+// In general identifiers that pass this validation should be safe for use as filesystem path components.
+func Validate(s string) error {
+	if len(s) == 0 {
+		return errors.Wrapf(errdefs.ErrInvalidArgument, "identifier must not be empty")
+	}
+
+	if len(s) > maxLength {
+		return errors.Wrapf(errdefs.ErrInvalidArgument, "identifier %q greater than maximum length (%d characters)", s, maxLength)
+	}
+
+	if !identifierRe.MatchString(s) {
+		return errors.Wrapf(errdefs.ErrInvalidArgument, "identifier %q must match %v", s, identifierRe)
+	}
+	return nil
+}
+
+func reGroup(s string) string {
+	return `(?:` + s + `)`
+}
+
+func reAnchor(s string) string {
+	return `^` + s + `$`
+}

vendor/github.com/containerd/containerd/log/context.go

@@ -0,0 +1,68 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package log
+
+import (
+	"context"
+
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	// G is an alias for GetLogger.
+	//
+	// We may want to define this locally to a package to get package tagged log
+	// messages.
+	G = GetLogger
+
+	// L is an alias for the standard logger.
+	L = logrus.NewEntry(logrus.StandardLogger())
+)
+
+type (
+	loggerKey struct{}
+)
+
+const (
+	// RFC3339NanoFixed is time.RFC3339Nano with nanoseconds padded using zeros to
+	// ensure the formatted time is always the same number of characters.
+	RFC3339NanoFixed = "2006-01-02T15:04:05.000000000Z07:00"
+
+	// TextFormat represents the text logging format
+	TextFormat = "text"
+
+	// JSONFormat represents the JSON logging format
+	JSONFormat = "json"
+)
+
+// WithLogger returns a new context with the provided logger. Use in
+// combination with logger.WithField(s) for great effect.
+func WithLogger(ctx context.Context, logger *logrus.Entry) context.Context {
+	return context.WithValue(ctx, loggerKey{}, logger)
+}
+
+// GetLogger retrieves the current logger from the context. If no logger is
+// available, the default logger is returned.
+func GetLogger(ctx context.Context) *logrus.Entry {
+	logger := ctx.Value(loggerKey{})
+
+	if logger == nil {
+		return L
+	}
+
+	return logger.(*logrus.Entry)
+}

vendor/github.com/containerd/containerd/namespaces/context.go

@@ -0,0 +1,78 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package namespaces
+
+import (
+	"context"
+	"os"
+
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/identifiers"
+	"github.com/pkg/errors"
+)
+
+const (
+	// NamespaceEnvVar is the environment variable key name
+	NamespaceEnvVar = "CONTAINERD_NAMESPACE"
+	// Default is the name of the default namespace
+	Default = "default"
+)
+
+type namespaceKey struct{}
+
+// WithNamespace sets a given namespace on the context
+func WithNamespace(ctx context.Context, namespace string) context.Context {
+	ctx = context.WithValue(ctx, namespaceKey{}, namespace) // set our key for namespace
+	// also store on the grpc and ttrpc headers so it gets picked up by any clients that
+	// are using this.
+	return withTTRPCNamespaceHeader(withGRPCNamespaceHeader(ctx, namespace), namespace)
+}
+
+// NamespaceFromEnv uses the namespace defined in CONTAINERD_NAMESPACE or
+// default
+func NamespaceFromEnv(ctx context.Context) context.Context {
+	namespace := os.Getenv(NamespaceEnvVar)
+	if namespace == "" {
+		namespace = Default
+	}
+	return WithNamespace(ctx, namespace)
+}
+
+// Namespace returns the namespace from the context.
+//
+// The namespace is not guaranteed to be valid.
+func Namespace(ctx context.Context) (string, bool) {
+	namespace, ok := ctx.Value(namespaceKey{}).(string)
+	if !ok {
+		if namespace, ok = fromGRPCHeader(ctx); !ok {
+			return fromTTRPCHeader(ctx)
+		}
+	}
+	return namespace, ok
+}
+
+// NamespaceRequired returns the valid namespace from the context or an error.
+func NamespaceRequired(ctx context.Context) (string, error) {
+	namespace, ok := Namespace(ctx)
+	if !ok || namespace == "" {
+		return "", errors.Wrapf(errdefs.ErrFailedPrecondition, "namespace is required")
+	}
+	if err := identifiers.Validate(namespace); err != nil {
+		return "", errors.Wrap(err, "namespace validation")
+	}
+	return namespace, nil
+}

vendor/github.com/containerd/containerd/namespaces/grpc.go

@@ -0,0 +1,61 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package namespaces
+
+import (
+	"context"
+
+	"google.golang.org/grpc/metadata"
+)
+
+const (
+	// GRPCHeader defines the header name for specifying a containerd namespace.
+	GRPCHeader = "containerd-namespace"
+)
+
+// NOTE(stevvooe): We can stub this file out if we don't want a grpc dependency here.
+
+func withGRPCNamespaceHeader(ctx context.Context, namespace string) context.Context {
+	// also store on the grpc headers so it gets picked up by any clients that
+	// are using this.
+	nsheader := metadata.Pairs(GRPCHeader, namespace)
+	md, ok := metadata.FromOutgoingContext(ctx) // merge with outgoing context.
+	if !ok {
+		md = nsheader
+	} else {
+		// order ensures the latest is first in this list.
+		md = metadata.Join(nsheader, md)
+	}
+
+	return metadata.NewOutgoingContext(ctx, md)
+}
+
+func fromGRPCHeader(ctx context.Context) (string, bool) {
+	// try to extract for use in grpc servers.
+	md, ok := metadata.FromIncomingContext(ctx)
+	if !ok {
+		// TODO(stevvooe): Check outgoing context?
+		return "", false
+	}
+
+	values := md[GRPCHeader]
+	if len(values) == 0 {
+		return "", false
+	}
+
+	return values[0], true
+}

vendor/github.com/containerd/containerd/namespaces/store.go

@@ -0,0 +1,46 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package namespaces
+
+import "context"
+
+// Store provides introspection about namespaces.
+//
+// Note that these are slightly different than other objects, which are record
+// oriented. A namespace is really just a name and a set of labels. Objects
+// that belong to a namespace are returned when the namespace is assigned to a
+// given context.
+//
+//
+type Store interface {
+	Create(ctx context.Context, namespace string, labels map[string]string) error
+	Labels(ctx context.Context, namespace string) (map[string]string, error)
+	SetLabel(ctx context.Context, namespace, key, value string) error
+	List(ctx context.Context) ([]string, error)
+
+	// Delete removes the namespace. The namespace must be empty to be deleted.
+	Delete(ctx context.Context, namespace string, opts ...DeleteOpts) error
+}
+
+// DeleteInfo specifies information for the deletion of a namespace
+type DeleteInfo struct {
+	// Name of the namespace
+	Name string
+}
+
+// DeleteOpts allows the caller to set options for namespace deletion
+type DeleteOpts func(context.Context, *DeleteInfo) error

vendor/github.com/containerd/containerd/namespaces/ttrpc.go

@@ -0,0 +1,51 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package namespaces
+
+import (
+	"context"
+
+	"github.com/containerd/ttrpc"
+)
+
+const (
+	// TTRPCHeader defines the header name for specifying a containerd namespace
+	TTRPCHeader = "containerd-namespace-ttrpc"
+)
+
+func copyMetadata(src ttrpc.MD) ttrpc.MD {
+	md := ttrpc.MD{}
+	for k, v := range src {
+		md[k] = append(md[k], v...)
+	}
+	return md
+}
+
+func withTTRPCNamespaceHeader(ctx context.Context, namespace string) context.Context {
+	md, ok := ttrpc.GetMetadata(ctx)
+	if !ok {
+		md = ttrpc.MD{}
+	} else {
+		md = copyMetadata(md)
+	}
+	md.Set(TTRPCHeader, namespace)
+	return ttrpc.WithMetadata(ctx, md)
+}
+
+func fromTTRPCHeader(ctx context.Context) (string, bool) {
+	return ttrpc.GetMetadataValue(ctx, TTRPCHeader)
+}

vendor/github.com/containerd/containerd/plugin/context.go

@@ -0,0 +1,146 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package plugin
+
+import (
+	"context"
+	"path/filepath"
+
+	"github.com/containerd/containerd/errdefs"
+	"github.com/containerd/containerd/events/exchange"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+	"github.com/pkg/errors"
+)
+
+// InitContext is used for plugin inititalization
+type InitContext struct {
+	Context      context.Context
+	Root         string
+	State        string
+	Config       interface{}
+	Address      string
+	TTRPCAddress string
+	Events       *exchange.Exchange
+
+	Meta *Meta // plugins can fill in metadata at init.
+
+	plugins *Set
+}
+
+// NewContext returns a new plugin InitContext
+func NewContext(ctx context.Context, r *Registration, plugins *Set, root, state string) *InitContext {
+	return &InitContext{
+		Context: ctx,
+		Root:    filepath.Join(root, r.URI()),
+		State:   filepath.Join(state, r.URI()),
+		Meta: &Meta{
+			Exports: map[string]string{},
+		},
+		plugins: plugins,
+	}
+}
+
+// Get returns the first plugin by its type
+func (i *InitContext) Get(t Type) (interface{}, error) {
+	return i.plugins.Get(t)
+}
+
+// Meta contains information gathered from the registration and initialization
+// process.
+type Meta struct {
+	Platforms    []ocispec.Platform // platforms supported by plugin
+	Exports      map[string]string  // values exported by plugin
+	Capabilities []string           // feature switches for plugin
+}
+
+// Plugin represents an initialized plugin, used with an init context.
+type Plugin struct {
+	Registration *Registration // registration, as initialized
+	Config       interface{}   // config, as initialized
+	Meta         *Meta
+
+	instance interface{}
+	err      error // will be set if there was an error initializing the plugin
+}
+
+// Err returns the errors during initialization.
+// returns nil if not error was encountered
+func (p *Plugin) Err() error {
+	return p.err
+}
+
+// Instance returns the instance and any initialization error of the plugin
+func (p *Plugin) Instance() (interface{}, error) {
+	return p.instance, p.err
+}
+
+// Set defines a plugin collection, used with InitContext.
+//
+// This maintains ordering and unique indexing over the set.
+//
+// After iteratively instantiating plugins, this set should represent, the
+// ordered, initialization set of plugins for a containerd instance.
+type Set struct {
+	ordered     []*Plugin // order of initialization
+	byTypeAndID map[Type]map[string]*Plugin
+}
+
+// NewPluginSet returns an initialized plugin set
+func NewPluginSet() *Set {
+	return &Set{
+		byTypeAndID: make(map[Type]map[string]*Plugin),
+	}
+}
+
+// Add a plugin to the set
+func (ps *Set) Add(p *Plugin) error {
+	if byID, typeok := ps.byTypeAndID[p.Registration.Type]; !typeok {
+		ps.byTypeAndID[p.Registration.Type] = map[string]*Plugin{
+			p.Registration.ID: p,
+		}
+	} else if _, idok := byID[p.Registration.ID]; !idok {
+		byID[p.Registration.ID] = p
+	} else {
+		return errors.Wrapf(errdefs.ErrAlreadyExists, "plugin %v already initialized", p.Registration.URI())
+	}
+
+	ps.ordered = append(ps.ordered, p)
+	return nil
+}
+
+// Get returns the first plugin by its type
+func (ps *Set) Get(t Type) (interface{}, error) {
+	for _, v := range ps.byTypeAndID[t] {
+		return v.Instance()
+	}
+	return nil, errors.Wrapf(errdefs.ErrNotFound, "no plugins registered for %s", t)
+}
+
+// GetAll plugins in the set
+func (i *InitContext) GetAll() []*Plugin {
+	return i.plugins.ordered
+}
+
+// GetByType returns all plugins with the specific type.
+func (i *InitContext) GetByType(t Type) (map[string]*Plugin, error) {
+	p, ok := i.plugins.byTypeAndID[t]
+	if !ok {
+		return nil, errors.Wrapf(errdefs.ErrNotFound, "no plugins registered for %s", t)
+	}
+
+	return p, nil
+}

vendor/github.com/containerd/containerd/plugin/plugin.go

@@ -0,0 +1,239 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package plugin
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/containerd/ttrpc"
+	"github.com/pkg/errors"
+	"google.golang.org/grpc"
+)
+
+var (
+	// ErrNoType is returned when no type is specified
+	ErrNoType = errors.New("plugin: no type")
+	// ErrNoPluginID is returned when no id is specified
+	ErrNoPluginID = errors.New("plugin: no id")
+	// ErrIDRegistered is returned when a duplicate id is already registered
+	ErrIDRegistered = errors.New("plugin: id already registered")
+	// ErrSkipPlugin is used when a plugin is not initialized and should not be loaded,
+	// this allows the plugin loader differentiate between a plugin which is configured
+	// not to load and one that fails to load.
+	ErrSkipPlugin = errors.New("skip plugin")
+
+	// ErrInvalidRequires will be thrown if the requirements for a plugin are
+	// defined in an invalid manner.
+	ErrInvalidRequires = errors.New("invalid requires")
+)
+
+// IsSkipPlugin returns true if the error is skipping the plugin
+func IsSkipPlugin(err error) bool {
+	return errors.Is(err, ErrSkipPlugin)
+}
+
+// Type is the type of the plugin
+type Type string
+
+func (t Type) String() string { return string(t) }
+
+const (
+	// InternalPlugin implements an internal plugin to containerd
+	InternalPlugin Type = "io.containerd.internal.v1"
+	// RuntimePlugin implements a runtime
+	RuntimePlugin Type = "io.containerd.runtime.v1"
+	// RuntimePluginV2 implements a runtime v2
+	RuntimePluginV2 Type = "io.containerd.runtime.v2"
+	// ServicePlugin implements a internal service
+	ServicePlugin Type = "io.containerd.service.v1"
+	// GRPCPlugin implements a grpc service
+	GRPCPlugin Type = "io.containerd.grpc.v1"
+	// SnapshotPlugin implements a snapshotter
+	SnapshotPlugin Type = "io.containerd.snapshotter.v1"
+	// TaskMonitorPlugin implements a task monitor
+	TaskMonitorPlugin Type = "io.containerd.monitor.v1"
+	// DiffPlugin implements a differ
+	DiffPlugin Type = "io.containerd.differ.v1"
+	// MetadataPlugin implements a metadata store
+	MetadataPlugin Type = "io.containerd.metadata.v1"
+	// ContentPlugin implements a content store
+	ContentPlugin Type = "io.containerd.content.v1"
+	// GCPlugin implements garbage collection policy
+	GCPlugin Type = "io.containerd.gc.v1"
+)
+
+const (
+	// RuntimeLinuxV1 is the legacy linux runtime
+	RuntimeLinuxV1 = "io.containerd.runtime.v1.linux"
+	// RuntimeRuncV1 is the runc runtime that supports a single container
+	RuntimeRuncV1 = "io.containerd.runc.v1"
+	// RuntimeRuncV2 is the runc runtime that supports multiple containers per shim
+	RuntimeRuncV2 = "io.containerd.runc.v2"
+)
+
+// Registration contains information for registering a plugin
+type Registration struct {
+	// Type of the plugin
+	Type Type
+	// ID of the plugin
+	ID string
+	// Config specific to the plugin
+	Config interface{}
+	// Requires is a list of plugins that the registered plugin requires to be available
+	Requires []Type
+
+	// InitFn is called when initializing a plugin. The registration and
+	// context are passed in. The init function may modify the registration to
+	// add exports, capabilities and platform support declarations.
+	InitFn func(*InitContext) (interface{}, error)
+	// Disable the plugin from loading
+	Disable bool
+}
+
+// Init the registered plugin
+func (r *Registration) Init(ic *InitContext) *Plugin {
+	p, err := r.InitFn(ic)
+	return &Plugin{
+		Registration: r,
+		Config:       ic.Config,
+		Meta:         ic.Meta,
+		instance:     p,
+		err:          err,
+	}
+}
+
+// URI returns the full plugin URI
+func (r *Registration) URI() string {
+	return fmt.Sprintf("%s.%s", r.Type, r.ID)
+}
+
+// Service allows GRPC services to be registered with the underlying server
+type Service interface {
+	Register(*grpc.Server) error
+}
+
+// TTRPCService allows TTRPC services to be registered with the underlying server
+type TTRPCService interface {
+	RegisterTTRPC(*ttrpc.Server) error
+}
+
+// TCPService allows GRPC services to be registered with the underlying tcp server
+type TCPService interface {
+	RegisterTCP(*grpc.Server) error
+}
+
+var register = struct {
+	sync.RWMutex
+	r []*Registration
+}{}
+
+// Load loads all plugins at the provided path into containerd
+func Load(path string) (err error) {
+	defer func() {
+		if v := recover(); v != nil {
+			rerr, ok := v.(error)
+			if !ok {
+				rerr = fmt.Errorf("%s", v)
+			}
+			err = rerr
+		}
+	}()
+	return loadPlugins(path)
+}
+
+// Register allows plugins to register
+func Register(r *Registration) {
+	register.Lock()
+	defer register.Unlock()
+
+	if r.Type == "" {
+		panic(ErrNoType)
+	}
+	if r.ID == "" {
+		panic(ErrNoPluginID)
+	}
+	if err := checkUnique(r); err != nil {
+		panic(err)
+	}
+
+	var last bool
+	for _, requires := range r.Requires {
+		if requires == "*" {
+			last = true
+		}
+	}
+	if last && len(r.Requires) != 1 {
+		panic(ErrInvalidRequires)
+	}
+
+	register.r = append(register.r, r)
+}
+
+func checkUnique(r *Registration) error {
+	for _, registered := range register.r {
+		if r.URI() == registered.URI() {
+			return errors.Wrap(ErrIDRegistered, r.URI())
+		}
+	}
+	return nil
+}
+
+// DisableFilter filters out disabled plugins
+type DisableFilter func(r *Registration) bool
+
+// Graph returns an ordered list of registered plugins for initialization.
+// Plugins in disableList specified by id will be disabled.
+func Graph(filter DisableFilter) (ordered []*Registration) {
+	register.RLock()
+	defer register.RUnlock()
+
+	for _, r := range register.r {
+		if filter(r) {
+			r.Disable = true
+		}
+	}
+
+	added := map[*Registration]bool{}
+	for _, r := range register.r {
+		if r.Disable {
+			continue
+		}
+		children(r, added, &ordered)
+		if !added[r] {
+			ordered = append(ordered, r)
+			added[r] = true
+		}
+	}
+	return ordered
+}
+
+func children(reg *Registration, added map[*Registration]bool, ordered *[]*Registration) {
+	for _, t := range reg.Requires {
+		for _, r := range register.r {
+			if !r.Disable &&
+				r.URI() != reg.URI() &&
+				(t == "*" || r.Type == t) {
+				children(r, added, ordered)
+				if !added[r] {
+					*ordered = append(*ordered, r)
+					added[r] = true
+				}
+			}
+		}
+	}
+}

vendor/github.com/containerd/containerd/plugin/plugin_go18.go

@@ -0,0 +1,62 @@
+// +build go1.8,!windows,amd64,!static_build,!gccgo
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package plugin
+
+import (
+	"fmt"
+	"path/filepath"
+	"plugin"
+	"runtime"
+)
+
+// loadPlugins loads all plugins for the OS and Arch
+// that containerd is built for inside the provided path
+func loadPlugins(path string) error {
+	abs, err := filepath.Abs(path)
+	if err != nil {
+		return err
+	}
+	pattern := filepath.Join(abs, fmt.Sprintf(
+		"*-%s-%s.%s",
+		runtime.GOOS,
+		runtime.GOARCH,
+		getLibExt(),
+	))
+	libs, err := filepath.Glob(pattern)
+	if err != nil {
+		return err
+	}
+	for _, lib := range libs {
+		if _, err := plugin.Open(lib); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+// getLibExt returns a platform specific lib extension for
+// the platform that containerd is running on
+func getLibExt() string {
+	switch runtime.GOOS {
+	case "windows":
+		return "dll"
+	default:
+		return "so"
+	}
+}

vendor/github.com/containerd/containerd/plugin/plugin_other.go

@@ -0,0 +1,24 @@
+// +build !go1.8 windows !amd64 static_build gccgo
+
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package plugin
+
+func loadPlugins(path string) error {
+	// plugins not supported until 1.8
+	return nil
+}

vendor/github.com/containerd/ttrpc/.gitignore

@@ -0,0 +1,14 @@
+# Binaries for programs and plugins
+*.exe
+*.dll
+*.so
+*.dylib
+
+# Test binary, build with `go test -c`
+*.test
+
+# Output of the go coverage tool, specifically when used with LiteIDE
+*.out
+
+# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736
+.glide/

vendor/github.com/containerd/ttrpc/.travis.yml

@@ -0,0 +1,24 @@
+dist: bionic
+language: go
+
+go:
+  - "1.13.x"
+  - "1.15.x"
+
+install:
+  # Don't change local go.{mod, sum} by go get tools.
+  #
+  # ref: https://github.com/golang/go/issues/27643
+  - pushd ..; go get -u github.com/vbatts/git-validation; popd
+  - pushd ..; go get -u github.com/kunalkushwaha/ltag; popd
+
+before_script:
+  - pushd ..; git clone https://github.com/containerd/project; popd
+
+script:
+  - DCO_VERBOSITY=-q ../project/script/validate/dco
+  - ../project/script/validate/fileheader ../project/
+  - go test -v -race -covermode=atomic -coverprofile=coverage.txt ./...
+
+after_success:
+  - bash <(curl -s https://codecov.io/bash)

vendor/github.com/containerd/ttrpc/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/containerd/ttrpc/README.md

@@ -0,0 +1,62 @@
+# ttrpc
+
+[![Build Status](https://travis-ci.org/containerd/ttrpc.svg?branch=master)](https://travis-ci.org/containerd/ttrpc)
+
+GRPC for low-memory environments.
+
+The existing grpc-go project requires a lot of memory overhead for importing
+packages and at runtime. While this is great for many services with low density
+requirements, this can be a problem when running a large number of services on
+a single machine or on a machine with a small amount of memory.
+
+Using the same GRPC definitions, this project reduces the binary size and
+protocol overhead required. We do this by eliding the `net/http`, `net/http2`
+and `grpc` package used by grpc replacing it with a lightweight framing
+protocol. The result are smaller binaries that use less resident memory with
+the same ease of use as GRPC.
+
+Please note that while this project supports generating either end of the
+protocol, the generated service definitions will be incompatible with regular
+GRPC services, as they do not speak the same protocol.
+
+# Usage
+
+Create a gogo vanity binary (see
+[`cmd/protoc-gen-gogottrpc/main.go`](cmd/protoc-gen-gogottrpc/main.go) for an
+example with the ttrpc plugin enabled.
+
+It's recommended to use [`protobuild`](https://github.com//stevvooe/protobuild)
+to build the protobufs for this project, but this will work with protoc
+directly, if required.
+
+# Differences from GRPC
+
+- The protocol stack has been replaced with a lighter protocol that doesn't
+  require http, http2 and tls.
+- The client and server interface are identical whereas in GRPC there is a
+  client and server interface that are different.
+- The Go stdlib context package is used instead.
+- No support for streams yet.
+
+# Status
+
+Very new. YMMV.
+
+TODO:
+
+- [X] Plumb error codes and GRPC status
+- [X] Remove use of any type and dependency on typeurl package
+- [X] Ensure that protocol can support streaming in the future
+- [ ] Document protocol layout
+- [ ] Add testing under concurrent load to ensure
+- [ ] Verify connection error handling
+
+# Project details
+
+ttrpc is a containerd sub-project, licensed under the [Apache 2.0 license](./LICENSE).
+As a containerd sub-project, you will find the:
+ * [Project governance](https://github.com/containerd/project/blob/master/GOVERNANCE.md),
+ * [Maintainers](https://github.com/containerd/project/blob/master/MAINTAINERS),
+ * and [Contributing guidelines](https://github.com/containerd/project/blob/master/CONTRIBUTING.md)
+
+information in our [`containerd/project`](https://github.com/containerd/project) repository.

vendor/github.com/containerd/ttrpc/channel.go

@@ -0,0 +1,153 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+import (
+	"bufio"
+	"encoding/binary"
+	"io"
+	"net"
+	"sync"
+
+	"github.com/pkg/errors"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+const (
+	messageHeaderLength = 10
+	messageLengthMax    = 4 << 20
+)
+
+type messageType uint8
+
+const (
+	messageTypeRequest  messageType = 0x1
+	messageTypeResponse messageType = 0x2
+)
+
+// messageHeader represents the fixed-length message header of 10 bytes sent
+// with every request.
+type messageHeader struct {
+	Length   uint32      // length excluding this header. b[:4]
+	StreamID uint32      // identifies which request stream message is a part of. b[4:8]
+	Type     messageType // message type b[8]
+	Flags    uint8       // reserved          b[9]
+}
+
+func readMessageHeader(p []byte, r io.Reader) (messageHeader, error) {
+	_, err := io.ReadFull(r, p[:messageHeaderLength])
+	if err != nil {
+		return messageHeader{}, err
+	}
+
+	return messageHeader{
+		Length:   binary.BigEndian.Uint32(p[:4]),
+		StreamID: binary.BigEndian.Uint32(p[4:8]),
+		Type:     messageType(p[8]),
+		Flags:    p[9],
+	}, nil
+}
+
+func writeMessageHeader(w io.Writer, p []byte, mh messageHeader) error {
+	binary.BigEndian.PutUint32(p[:4], mh.Length)
+	binary.BigEndian.PutUint32(p[4:8], mh.StreamID)
+	p[8] = byte(mh.Type)
+	p[9] = mh.Flags
+
+	_, err := w.Write(p[:])
+	return err
+}
+
+var buffers sync.Pool
+
+type channel struct {
+	conn  net.Conn
+	bw    *bufio.Writer
+	br    *bufio.Reader
+	hrbuf [messageHeaderLength]byte // avoid alloc when reading header
+	hwbuf [messageHeaderLength]byte
+}
+
+func newChannel(conn net.Conn) *channel {
+	return &channel{
+		conn: conn,
+		bw:   bufio.NewWriter(conn),
+		br:   bufio.NewReader(conn),
+	}
+}
+
+// recv a message from the channel. The returned buffer contains the message.
+//
+// If a valid grpc status is returned, the message header
+// returned will be valid and caller should send that along to
+// the correct consumer. The bytes on the underlying channel
+// will be discarded.
+func (ch *channel) recv() (messageHeader, []byte, error) {
+	mh, err := readMessageHeader(ch.hrbuf[:], ch.br)
+	if err != nil {
+		return messageHeader{}, nil, err
+	}
+
+	if mh.Length > uint32(messageLengthMax) {
+		if _, err := ch.br.Discard(int(mh.Length)); err != nil {
+			return mh, nil, errors.Wrapf(err, "failed to discard after receiving oversized message")
+		}
+
+		return mh, nil, status.Errorf(codes.ResourceExhausted, "message length %v exceed maximum message size of %v", mh.Length, messageLengthMax)
+	}
+
+	p := ch.getmbuf(int(mh.Length))
+	if _, err := io.ReadFull(ch.br, p); err != nil {
+		return messageHeader{}, nil, errors.Wrapf(err, "failed reading message")
+	}
+
+	return mh, p, nil
+}
+
+func (ch *channel) send(streamID uint32, t messageType, p []byte) error {
+	if err := writeMessageHeader(ch.bw, ch.hwbuf[:], messageHeader{Length: uint32(len(p)), StreamID: streamID, Type: t}); err != nil {
+		return err
+	}
+
+	_, err := ch.bw.Write(p)
+	if err != nil {
+		return err
+	}
+
+	return ch.bw.Flush()
+}
+
+func (ch *channel) getmbuf(size int) []byte {
+	// we can't use the standard New method on pool because we want to allocate
+	// based on size.
+	b, ok := buffers.Get().(*[]byte)
+	if !ok || cap(*b) < size {
+		// TODO(stevvooe): It may be better to allocate these in fixed length
+		// buckets to reduce fragmentation but its not clear that would help
+		// with performance. An ilogb approach or similar would work well.
+		bb := make([]byte, size)
+		b = &bb
+	} else {
+		*b = (*b)[:size]
+	}
+	return *b
+}
+
+func (ch *channel) putmbuf(p []byte) {
+	buffers.Put(&p)
+}

vendor/github.com/containerd/ttrpc/client.go

@@ -0,0 +1,368 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+import (
+	"context"
+	"io"
+	"net"
+	"os"
+	"strings"
+	"sync"
+	"syscall"
+	"time"
+
+	"github.com/gogo/protobuf/proto"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"google.golang.org/grpc/codes"
+	"google.golang.org/grpc/status"
+)
+
+// ErrClosed is returned by client methods when the underlying connection is
+// closed.
+var ErrClosed = errors.New("ttrpc: closed")
+
+// Client for a ttrpc server
+type Client struct {
+	codec   codec
+	conn    net.Conn
+	channel *channel
+	calls   chan *callRequest
+
+	ctx    context.Context
+	closed func()
+
+	closeOnce       sync.Once
+	userCloseFunc   func()
+	userCloseWaitCh chan struct{}
+
+	errOnce     sync.Once
+	err         error
+	interceptor UnaryClientInterceptor
+}
+
+// ClientOpts configures a client
+type ClientOpts func(c *Client)
+
+// WithOnClose sets the close func whenever the client's Close() method is called
+func WithOnClose(onClose func()) ClientOpts {
+	return func(c *Client) {
+		c.userCloseFunc = onClose
+	}
+}
+
+// WithUnaryClientInterceptor sets the provided client interceptor
+func WithUnaryClientInterceptor(i UnaryClientInterceptor) ClientOpts {
+	return func(c *Client) {
+		c.interceptor = i
+	}
+}
+
+func NewClient(conn net.Conn, opts ...ClientOpts) *Client {
+	ctx, cancel := context.WithCancel(context.Background())
+	c := &Client{
+		codec:           codec{},
+		conn:            conn,
+		channel:         newChannel(conn),
+		calls:           make(chan *callRequest),
+		closed:          cancel,
+		ctx:             ctx,
+		userCloseFunc:   func() {},
+		userCloseWaitCh: make(chan struct{}),
+		interceptor:     defaultClientInterceptor,
+	}
+
+	for _, o := range opts {
+		o(c)
+	}
+
+	go c.run()
+	return c
+}
+
+type callRequest struct {
+	ctx  context.Context
+	req  *Request
+	resp *Response  // response will be written back here
+	errs chan error // error written here on completion
+}
+
+func (c *Client) Call(ctx context.Context, service, method string, req, resp interface{}) error {
+	payload, err := c.codec.Marshal(req)
+	if err != nil {
+		return err
+	}
+
+	var (
+		creq = &Request{
+			Service: service,
+			Method:  method,
+			Payload: payload,
+		}
+
+		cresp = &Response{}
+	)
+
+	if metadata, ok := GetMetadata(ctx); ok {
+		metadata.setRequest(creq)
+	}
+
+	if dl, ok := ctx.Deadline(); ok {
+		creq.TimeoutNano = dl.Sub(time.Now()).Nanoseconds()
+	}
+
+	info := &UnaryClientInfo{
+		FullMethod: fullPath(service, method),
+	}
+	if err := c.interceptor(ctx, creq, cresp, info, c.dispatch); err != nil {
+		return err
+	}
+
+	if err := c.codec.Unmarshal(cresp.Payload, resp); err != nil {
+		return err
+	}
+
+	if cresp.Status != nil && cresp.Status.Code != int32(codes.OK) {
+		return status.ErrorProto(cresp.Status)
+	}
+	return nil
+}
+
+func (c *Client) dispatch(ctx context.Context, req *Request, resp *Response) error {
+	errs := make(chan error, 1)
+	call := &callRequest{
+		ctx:  ctx,
+		req:  req,
+		resp: resp,
+		errs: errs,
+	}
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case c.calls <- call:
+	case <-c.ctx.Done():
+		return c.error()
+	}
+
+	select {
+	case <-ctx.Done():
+		return ctx.Err()
+	case err := <-errs:
+		return filterCloseErr(err)
+	case <-c.ctx.Done():
+		return c.error()
+	}
+}
+
+func (c *Client) Close() error {
+	c.closeOnce.Do(func() {
+		c.closed()
+	})
+	return nil
+}
+
+// UserOnCloseWait is used to blocks untils the user's on-close callback
+// finishes.
+func (c *Client) UserOnCloseWait(ctx context.Context) error {
+	select {
+	case <-c.userCloseWaitCh:
+		return nil
+	case <-ctx.Done():
+		return ctx.Err()
+	}
+}
+
+type message struct {
+	messageHeader
+	p   []byte
+	err error
+}
+
+type receiver struct {
+	wg       *sync.WaitGroup
+	messages chan *message
+	err      error
+}
+
+func (r *receiver) run(ctx context.Context, c *channel) {
+	defer r.wg.Done()
+
+	for {
+		select {
+		case <-ctx.Done():
+			r.err = ctx.Err()
+			return
+		default:
+			mh, p, err := c.recv()
+			if err != nil {
+				_, ok := status.FromError(err)
+				if !ok {
+					// treat all errors that are not an rpc status as terminal.
+					// all others poison the connection.
+					r.err = filterCloseErr(err)
+					return
+				}
+			}
+			select {
+			case r.messages <- &message{
+				messageHeader: mh,
+				p:             p[:mh.Length],
+				err:           err,
+			}:
+			case <-ctx.Done():
+				r.err = ctx.Err()
+				return
+			}
+		}
+	}
+}
+
+func (c *Client) run() {
+	var (
+		streamID      uint32 = 1
+		waiters              = make(map[uint32]*callRequest)
+		calls                = c.calls
+		incoming             = make(chan *message)
+		receiversDone        = make(chan struct{})
+		wg            sync.WaitGroup
+	)
+
+	// broadcast the shutdown error to the remaining waiters.
+	abortWaiters := func(wErr error) {
+		for _, waiter := range waiters {
+			waiter.errs <- wErr
+		}
+	}
+	recv := &receiver{
+		wg:       &wg,
+		messages: incoming,
+	}
+	wg.Add(1)
+
+	go func() {
+		wg.Wait()
+		close(receiversDone)
+	}()
+	go recv.run(c.ctx, c.channel)
+
+	defer func() {
+		c.conn.Close()
+		c.userCloseFunc()
+		close(c.userCloseWaitCh)
+	}()
+
+	for {
+		select {
+		case call := <-calls:
+			if err := c.send(streamID, messageTypeRequest, call.req); err != nil {
+				call.errs <- err
+				continue
+			}
+
+			waiters[streamID] = call
+			streamID += 2 // enforce odd client initiated request ids
+		case msg := <-incoming:
+			call, ok := waiters[msg.StreamID]
+			if !ok {
+				logrus.Errorf("ttrpc: received message for unknown channel %v", msg.StreamID)
+				continue
+			}
+
+			call.errs <- c.recv(call.resp, msg)
+			delete(waiters, msg.StreamID)
+		case <-receiversDone:
+			// all the receivers have exited
+			if recv.err != nil {
+				c.setError(recv.err)
+			}
+			// don't return out, let the close of the context trigger the abort of waiters
+			c.Close()
+		case <-c.ctx.Done():
+			abortWaiters(c.error())
+			return
+		}
+	}
+}
+
+func (c *Client) error() error {
+	c.errOnce.Do(func() {
+		if c.err == nil {
+			c.err = ErrClosed
+		}
+	})
+	return c.err
+}
+
+func (c *Client) setError(err error) {
+	c.errOnce.Do(func() {
+		c.err = err
+	})
+}
+
+func (c *Client) send(streamID uint32, mtype messageType, msg interface{}) error {
+	p, err := c.codec.Marshal(msg)
+	if err != nil {
+		return err
+	}
+
+	return c.channel.send(streamID, mtype, p)
+}
+
+func (c *Client) recv(resp *Response, msg *message) error {
+	if msg.err != nil {
+		return msg.err
+	}
+
+	if msg.Type != messageTypeResponse {
+		return errors.New("unknown message type received")
+	}
+
+	defer c.channel.putmbuf(msg.p)
+	return proto.Unmarshal(msg.p, resp)
+}
+
+// filterCloseErr rewrites EOF and EPIPE errors to ErrClosed. Use when
+// returning from call or handling errors from main read loop.
+//
+// This purposely ignores errors with a wrapped cause.
+func filterCloseErr(err error) error {
+	switch {
+	case err == nil:
+		return nil
+	case err == io.EOF:
+		return ErrClosed
+	case errors.Cause(err) == io.EOF:
+		return ErrClosed
+	case strings.Contains(err.Error(), "use of closed network connection"):
+		return ErrClosed
+	default:
+		// if we have an epipe on a write or econnreset on a read , we cast to errclosed
+		var oerr *net.OpError
+		if errors.As(err, &oerr) && (oerr.Op == "write" || oerr.Op == "read") {
+			serr, sok := oerr.Err.(*os.SyscallError)
+			if sok && ((serr.Err == syscall.EPIPE && oerr.Op == "write") ||
+				(serr.Err == syscall.ECONNRESET && oerr.Op == "read")) {
+
+				return ErrClosed
+			}
+		}
+	}
+
+	return err
+}

vendor/github.com/containerd/ttrpc/codec.go

@@ -0,0 +1,42 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+import (
+	"github.com/gogo/protobuf/proto"
+	"github.com/pkg/errors"
+)
+
+type codec struct{}
+
+func (c codec) Marshal(msg interface{}) ([]byte, error) {
+	switch v := msg.(type) {
+	case proto.Message:
+		return proto.Marshal(v)
+	default:
+		return nil, errors.Errorf("ttrpc: cannot marshal unknown type: %T", msg)
+	}
+}
+
+func (c codec) Unmarshal(p []byte, msg interface{}) error {
+	switch v := msg.(type) {
+	case proto.Message:
+		return proto.Unmarshal(p, v)
+	default:
+		return errors.Errorf("ttrpc: cannot unmarshal into unknown type: %T", msg)
+	}
+}

vendor/github.com/containerd/ttrpc/config.go

@@ -0,0 +1,52 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package ttrpc
+
+import "github.com/pkg/errors"
+
+type serverConfig struct {
+	handshaker  Handshaker
+	interceptor UnaryServerInterceptor
+}
+
+// ServerOpt for configuring a ttrpc server
+type ServerOpt func(*serverConfig) error
+
+// WithServerHandshaker can be passed to NewServer to ensure that the
+// handshaker is called before every connection attempt.
+//
+// Only one handshaker is allowed per server.
+func WithServerHandshaker(handshaker Handshaker) ServerOpt {
+	return func(c *serverConfig) error {
+		if c.handshaker != nil {
+			return errors.New("only one handshaker allowed per server")
+		}
+		c.handshaker = handshaker
+		return nil
+	}
+}
+
+// WithUnaryServerInterceptor sets the provided interceptor on the server
+func WithUnaryServerInterceptor(i UnaryServerInterceptor) ServerOpt {
+	return func(c *serverConfig) error {
+		if c.interceptor != nil {
+			return errors.New("only one interceptor allowed per server")
+		}
+		c.interceptor = i
+		return nil
+	}
+}