Add CI to build toolkit-container image

This change adds CI definitions for building the toolkit-container
images. This modifies the existing CI and replaces the build-one
stage with multiple stages that do the following:
* peform the standard golang checks
* build the packages required by the images
* build the images for supported platforms
* releases the images (currently to the CI staging registry)

The build-all stage is included as a final step in the CI. This is
run after the release stage as the target platforms are not requried
from an imaging perspective. The build-all stage is only run on
MRs or tagged builds.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

.common-ci.yml

@@ -18,8 +18,7 @@ default:
       command: ["--experimental"]
 
 variables:
-  IMAGE: "${CI_REGISTRY_IMAGE}"
+  GIT_SUBMODULE_STRATEGY: recursive
-  IMAGE_TAG: "${CI_COMMIT_REF_SLUG}"
   BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}"
 
 stages:
@@ -28,10 +27,12 @@ stages:
   - go-checks
   - go-build
   - unit-tests
-  - build
+  - package-build
-  - build-long
+  - image-build
+  - test
   - scan
   - release
+  - build-all
 
 build-dev-image:
   stage: image
@@ -95,3 +96,314 @@ unit-tests:
   script:
     - make coverage
 
+
+# Define the distribution targets
+.dist-centos7:
+  variables:
+    DIST: centos7
+
+.dist-centos8:
+  variables:
+    DIST: centos8
+
+.dist-ubi8:
+  variables:
+    DIST: ubi8
+
+.dist-ubuntu18.04:
+  variables:
+    DIST: ubuntu18.04
+
+.arch-aarch64:
+  variables:
+    ARCH: aarch64
+
+.arch-amd64:
+  variables:
+    ARCH: amd64
+
+.arch-arm64:
+  variables:
+    ARCH: arm64
+
+.arch-ppc64le:
+  variables:
+    ARCH: ppc64le
+
+.arch-x86_64:
+  variables:
+    ARCH: x86_64
+
+# Define the package build helpers
+.multi-arch-build:
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+    - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes
+
+.package-artifacts:
+  variables:
+    ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}"
+    ARTIFACTS_ROOT: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}"
+    DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}"
+
+.package-build:
+  extends:
+    - .multi-arch-build
+    - .package-artifacts
+  stage: package-build
+  script:
+    - ./scripts/release.sh ${DIST}-${ARCH}
+
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
+
+# Define the package build targets
+package-ubuntu18.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-amd64
+
+package-ubuntu18.04-arm64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-arm64
+
+package-ubuntu18.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-ppc64le
+
+package-centos7-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-x86_64
+
+package-centos8-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-x86_64
+
+# Define the image build targets
+.image-build:
+  stage: image-build
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache bash make
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+
+image-centos7:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos7
+  needs:
+    - package-centos7-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-centos8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos8
+  needs:
+    - package-centos8-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-ubi8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubi8
+  needs:
+    # Note: The ubi8 image currently uses the centos7 packages
+    - package-centos7-x86_64
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-ubuntu18.04:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubuntu18.04
+  needs:
+    - package-ubuntu18.04-amd64
+    # TODO: These will be required once we generate multi-arch images
+    # - package-ubuntu18.04-arm64
+    # - package-ubuntu18.04-ppc64le
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+# Define test helpers
+.integration:
+  stage: test
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache make bash jq
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
+  script:
+    - make -f build/container/Makefile test-${DIST}
+
+.test:toolkit:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "toolkit"
+
+.test:docker:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "docker"
+
+.test:containerd:
+  # TODO: The containerd tests fail due to issues with SIGHUP.
+  # Until this is resolved with retry up to twice and allow failure here.
+  retry: 2
+  allow_failure: true
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "containerd"
+
+.test:crio:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "crio"
+
+# Define the test targets
+test-toolkit-ubuntu18.04:
+  extends:
+    - .test:toolkit
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-containerd-ubuntu18.04:
+  extends:
+    - .test:containerd
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-crio-ubuntu18.04:
+  extends:
+    - .test:crio
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-docker-ubuntu18.04:
+  extends:
+    - .test:docker
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+# .release forms the base of the deployment jobs which push images to the CI registry.
+# This is extended with the version to be deployed (e.g. the SHA or TAG) and the
+# target os.
+.release:
+  stage:
+    release
+  variables:
+    # Define the source image for the release
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    # OUT_IMAGE_VERSION is overridden for external releases
+    OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
+  stage: release
+  before_script:
+    # We ensure that the OUT_IMAGE_VERSION is set
+    - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
+
+    # In the case where we are deploying a different version to the CI_COMMIT_SHA, we
+    # need to tag the image.
+    # Note: a leading 'v' is stripped from the version if present
+    - apk add --no-cache make bash
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
+  script:
+    - docker tag "${IMAGE_NAME}:${VERSION}-${DIST}" "${OUT_IMAGE_NAME}:${OUT_IMAGE_VERSION}-${DIST}"
+    # Log in to the "output" registry, tag the image and push the image
+    - 'echo "Logging in to output registry ${OUT_REGISTRY}"'
+    - docker logout
+    - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}"
+    - make IMAGE_NAME=${OUT_IMAGE_NAME} VERSION=${OUT_IMAGE_VERSION} -f build/container/Makefile push-${DIST}
+
+# Define a staging release step that pushes an image to an internal "staging" repository
+# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
+# outside of the release process.
+.release:staging:
+  extends:
+    - .release
+  variables:
+    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    OUT_REGISTRY: "${CI_REGISTRY}"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit"
+
+# Define an external release step that pushes an image to an external repository.
+# This includes a devlopment image off master.
+.release:external:
+  extends:
+    - .release
+  rules:
+    - if: $CI_COMMIT_TAG
+      variables:
+        OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}"
+    - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH
+      variables:
+        OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
+
+# Define the release jobs
+release:staging-centos7:
+  extends:
+    - .release:staging
+    - .dist-centos7
+  needs:
+    - image-centos7
+
+release:staging-centos8:
+  extends:
+    - .release:staging
+    - .dist-centos8
+  needs:
+    - image-centos8
+
+release:staging-ubi8:
+  extends:
+    - .release:staging
+    - .dist-ubi8
+  needs:
+    - image-ubi8
+
+release:staging-ubuntu18.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu18.04
+  needs:
+    - test-toolkit-ubuntu18.04
+    - test-containerd-ubuntu18.04
+    - test-crio-ubuntu18.04
+    - test-docker-ubuntu18.04

.gitlab-ci.yml

@@ -15,40 +15,18 @@
 include:
   - .common-ci.yml
 
-.build-setup:
-  before_script:
-  - apk update
-  - apk upgrade
-  - apk add coreutils build-base sed git bash make
-  - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes
-
-# build-one jobs build packages for a single OS / ARCH combination.
-#
-# They are run during the first stage of the pipeline as a smoke test to ensure
-# that we can successfully build packages on all of our architectures for a
-# single OS. They are triggered on any change to an MR. No artifacts are
-# produced as part of build-one jobs.
-.build-one-setup:
-  extends:
-    - .build-setup
-  stage: build
-  rules:
-    - if: $CI_MERGE_REQUEST_ID
-
 # build-all jobs build packages for every OS / ARCH combination we support.
 #
 # They are run under two conditions:
 # 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0)
 # 2) Manually by a reviewer just before merging a MR.
-#
+.build-all-for-arch:
-# Unlike build-one jobs, it takes a long time to build the full suite
+  variables:
-# OS / ARCH combinations, so this is optimized to only run once per MR
+    # Setting DIST=docker invokes the docker- release targets
-# (assuming it all passes). A full set of artifacts including the packages
+    DIST: docker
-# built for each OS / ARCH are produced as a result of these jobs.
-.build-all-setup:
   extends:
-    - .build-setup
+    - .package-build
-  stage: build-long
+  stage: build-all
   timeout: 2h 30m
   rules:
     - if: $CI_COMMIT_TAG
@@ -56,66 +34,29 @@ include:
     - if: $CI_MERGE_REQUEST_ID
       when: always
 
-  variables:
-    ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}"
-    ARTIFACTS_DIR: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}"
-    DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_DIR}"
-
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_DIR}
-
-# The full set of build-one jobs organizes to build
-# ubuntu18.04 in parallel on each of our supported ARCHs.
-build-one-amd64:
-  extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-amd64
-  rules:
-    - when: always
-
-build-one-ppc64le:
-  extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-ppc64le
-
-build-one-arm64:
-  extends:
-    - .build-one-setup
-  script:
-    - make ubuntu18.04-arm64
-
 # The full set of build-all jobs organized to
 # have builds for each ARCH run in parallel.
 build-all-amd64:
   extends:
-    - .build-all-setup
+    - .build-all-for-arch
-  script:
+    - .arch-amd64
-    - make docker-amd64
 
 build-all-x86_64:
   extends:
-    - .build-all-setup
+    - .build-all-for-arch
-  script:
+    - .arch-x86_64
-    - make docker-x86_64
 
 build-all-ppc64le:
   extends:
-    - .build-all-setup
+    - .build-all-for-arch
-  script:
+    - .arch-ppc64le
-    - make docker-ppc64le
 
 build-all-arm64:
   extends:
-    - .build-all-setup
+    - .build-all-for-arch
-  script:
+    - .arch-arm64
-    - make docker-arm64
 
 build-all-aarch64:
   extends:
-    - .build-all-setup
+    - .build-all-for-arch
-  script:
+    - .arch-aarch64
-    - make docker-aarch64

build/container/Dockerfile.centos

@@ -29,14 +29,15 @@ ARG GOPATH=/artifacts
 # Install the experiemental nvidia-container-runtime
 # NOTE: This will be integrated into the nvidia-container-toolkit package / repo
 ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
 
+WORKDIR /build
 COPY . .
 
 # NOTE: Until the config utilities are properly integrated into the
 # nvidia-container-toolkit repository, these are built from the `tools` folder
 # and not `cmd`.
-RUN go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
 
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
@@ -48,7 +49,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES=utility
 WORKDIR /artifacts/packages
 
 ARG ARTIFACTS_DIR
-COPY ${ARTIFACTS_DIR}/* /artifacts/packages
+COPY ${ARTIFACTS_DIR}/* /artifacts/packages/
 
 ARG PACKAGE_VERSION
 RUN yum localinstall -y \

build/container/Dockerfile.ubuntu

@@ -29,14 +29,15 @@ ARG GOPATH=/artifacts
 # Install the experiemental nvidia-container-runtime
 # NOTE: This will be integrated into the nvidia-container-toolkit package / repo
 ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental
-RUN go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
+RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION}
 
+WORKDIR /build
 COPY . .
 
 # NOTE: Until the config utilities are properly integrated into the
 # nvidia-container-toolkit repository, these are built from the `tools` folder
 # and not `cmd`.
-RUN go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
+RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/...
 
 
 FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
@@ -54,7 +55,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES=utility
 WORKDIR /artifacts/packages
 
 ARG ARTIFACTS_DIR
-COPY ${ARTIFACTS_DIR}/* /artifacts/packages
+COPY ${ARTIFACTS_DIR}/* /artifacts/packages/
 
 ARG PACKAGE_VERSION
 RUN dpkg -i \
@@ -64,7 +65,7 @@ RUN dpkg -i \
 
 WORKDIR /work
 
-COPY --from=build /artifacts/bin /work
+COPY --from=build /artifacts/bin /work/
 
 ENV PATH=/work:$PATH
 

build/container/Makefile

@@ -40,7 +40,7 @@ DEFAULT_PUSH_TARGET := ubuntu18.04
 TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
 
 BUILD_TARGETS := $(patsubst %, build-%, $(TARGETS))
-# PUSH_TARGETS := $(patsubst %, push-%, $(TARGETS))
+PUSH_TARGETS := $(patsubst %, push-%, $(TARGETS))
 TEST_TARGETS := $(patsubst %, test-%, $(TARGETS))
 
 .PHONY: $(TARGETS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
@@ -76,8 +76,10 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_DIR)
 		$(CURDIR)
 
 
+ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
+
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
-build-ubuntu%: ARTIFACTS_DIR = dist/$(*)/amd64
+build-ubuntu%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/amd64
 build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 
 build-ubuntu18.04: BASE_DIST := ubuntu18.04
@@ -85,12 +87,12 @@ build-ubuntu20.04: BASE_DIST := ubuntu20.04
 
 build-ubi8: DOCKERFILE_SUFFIX := centos
 # TODO: Update this to use the centos8 packages
-build-ubi8: ARTIFACTS_DIR = dist/centos7/x86_64
+build-ubi8: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/centos7/x86_64
 build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 build-ubi8: BASE_DIST := ubi8
 
 build-centos%: DOCKERFILE_SUFFIX := centos
-build-centos%: ARTIFACTS_DIR = dist/$(*)/x86_64
+build-centos%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/x86_64
 build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-centos7: BASE_DIST := centos7