From c35444c76ca2a0926fc67a9909c7a011c2b6dcfc Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 13 Oct 2021 14:04:41 +0200 Subject: [PATCH] Add CI to build toolkit-container image This change adds CI definitions for building the toolkit-container images. This modifies the existing CI and replaces the build-one stage with multiple stages that do the following: * peform the standard golang checks * build the packages required by the images * build the images for supported platforms * releases the images (currently to the CI staging registry) The build-all stage is included as a final step in the CI. This is run after the release stage as the target platforms are not requried from an imaging perspective. The build-all stage is only run on MRs or tagged builds. Signed-off-by: Evan Lezar --- .common-ci.yml | 320 +++++++++++++++++++++++++++++- .gitlab-ci.yml | 91 ++------- build/container/Dockerfile.centos | 7 +- build/container/Dockerfile.ubuntu | 9 +- build/container/Makefile | 10 +- 5 files changed, 347 insertions(+), 90 deletions(-) diff --git a/.common-ci.yml b/.common-ci.yml index 369c06a2..3a4ce97c 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -18,8 +18,7 @@ default: command: ["--experimental"] variables: - IMAGE: "${CI_REGISTRY_IMAGE}" - IMAGE_TAG: "${CI_COMMIT_REF_SLUG}" + GIT_SUBMODULE_STRATEGY: recursive BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}" stages: @@ -28,10 +27,12 @@ stages: - go-checks - go-build - unit-tests - - build - - build-long + - package-build + - image-build + - test - scan - release + - build-all build-dev-image: stage: image @@ -95,3 +96,314 @@ unit-tests: script: - make coverage + +# Define the distribution targets +.dist-centos7: + variables: + DIST: centos7 + +.dist-centos8: + variables: + DIST: centos8 + +.dist-ubi8: + variables: + DIST: ubi8 + +.dist-ubuntu18.04: + variables: + DIST: ubuntu18.04 + +.arch-aarch64: + variables: + ARCH: aarch64 + +.arch-amd64: + variables: + ARCH: amd64 + +.arch-arm64: + variables: + ARCH: arm64 + +.arch-ppc64le: + variables: + ARCH: ppc64le + +.arch-x86_64: + variables: + ARCH: x86_64 + +# Define the package build helpers +.multi-arch-build: + before_script: + - apk add --no-cache coreutils build-base sed git bash make + - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes + +.package-artifacts: + variables: + ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}" + ARTIFACTS_ROOT: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}" + DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}" + +.package-build: + extends: + - .multi-arch-build + - .package-artifacts + stage: package-build + script: + - ./scripts/release.sh ${DIST}-${ARCH} + + artifacts: + name: ${ARTIFACTS_NAME} + paths: + - ${ARTIFACTS_ROOT} + +# Define the package build targets +package-ubuntu18.04-amd64: + extends: + - .package-build + - .dist-ubuntu18.04 + - .arch-amd64 + +package-ubuntu18.04-arm64: + extends: + - .package-build + - .dist-ubuntu18.04 + - .arch-arm64 + +package-ubuntu18.04-ppc64le: + extends: + - .package-build + - .dist-ubuntu18.04 + - .arch-ppc64le + +package-centos7-x86_64: + extends: + - .package-build + - .dist-centos7 + - .arch-x86_64 + +package-centos8-x86_64: + extends: + - .package-build + - .dist-centos8 + - .arch-x86_64 + +# Define the image build targets +.image-build: + stage: image-build + variables: + IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" + VERSION: "${CI_COMMIT_SHORT_SHA}" + before_script: + - apk add --no-cache bash make + - 'echo "Logging in to CI registry ${CI_REGISTRY}"' + - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" + +image-centos7: + extends: + - .image-build + - .package-artifacts + - .dist-centos7 + needs: + - package-centos7-x86_64 + script: + - make -f build/container/Makefile build-${DIST} + - make -f build/container/Makefile push-${DIST} + +image-centos8: + extends: + - .image-build + - .package-artifacts + - .dist-centos8 + needs: + - package-centos8-x86_64 + script: + - make -f build/container/Makefile build-${DIST} + - make -f build/container/Makefile push-${DIST} + +image-ubi8: + extends: + - .image-build + - .package-artifacts + - .dist-ubi8 + needs: + # Note: The ubi8 image currently uses the centos7 packages + - package-centos7-x86_64 + script: + - make -f build/container/Makefile build-${DIST} + - make -f build/container/Makefile push-${DIST} + +image-ubuntu18.04: + extends: + - .image-build + - .package-artifacts + - .dist-ubuntu18.04 + needs: + - package-ubuntu18.04-amd64 + # TODO: These will be required once we generate multi-arch images + # - package-ubuntu18.04-arm64 + # - package-ubuntu18.04-ppc64le + script: + - make -f build/container/Makefile build-${DIST} + - make -f build/container/Makefile push-${DIST} + +# Define test helpers +.integration: + stage: test + variables: + IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" + VERSION: "${CI_COMMIT_SHORT_SHA}" + before_script: + - apk add --no-cache make bash jq + - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" + - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}" + script: + - make -f build/container/Makefile test-${DIST} + +.test:toolkit: + extends: + - .integration + variables: + TEST_CASES: "toolkit" + +.test:docker: + extends: + - .integration + variables: + TEST_CASES: "docker" + +.test:containerd: + # TODO: The containerd tests fail due to issues with SIGHUP. + # Until this is resolved with retry up to twice and allow failure here. + retry: 2 + allow_failure: true + extends: + - .integration + variables: + TEST_CASES: "containerd" + +.test:crio: + extends: + - .integration + variables: + TEST_CASES: "crio" + +# Define the test targets +test-toolkit-ubuntu18.04: + extends: + - .test:toolkit + - .dist-ubuntu18.04 + needs: + - image-ubuntu18.04 + +test-containerd-ubuntu18.04: + extends: + - .test:containerd + - .dist-ubuntu18.04 + needs: + - image-ubuntu18.04 + +test-crio-ubuntu18.04: + extends: + - .test:crio + - .dist-ubuntu18.04 + needs: + - image-ubuntu18.04 + +test-docker-ubuntu18.04: + extends: + - .test:docker + - .dist-ubuntu18.04 + needs: + - image-ubuntu18.04 + +# .release forms the base of the deployment jobs which push images to the CI registry. +# This is extended with the version to be deployed (e.g. the SHA or TAG) and the +# target os. +.release: + stage: + release + variables: + # Define the source image for the release + IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" + VERSION: "${CI_COMMIT_SHORT_SHA}" + # OUT_IMAGE_VERSION is overridden for external releases + OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}" + stage: release + before_script: + # We ensure that the OUT_IMAGE_VERSION is set + - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1' + + # In the case where we are deploying a different version to the CI_COMMIT_SHA, we + # need to tag the image. + # Note: a leading 'v' is stripped from the version if present + - apk add --no-cache make bash + - 'echo "Logging in to CI registry ${CI_REGISTRY}"' + - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" + - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}" + script: + - docker tag "${IMAGE_NAME}:${VERSION}-${DIST}" "${OUT_IMAGE_NAME}:${OUT_IMAGE_VERSION}-${DIST}" + # Log in to the "output" registry, tag the image and push the image + - 'echo "Logging in to output registry ${OUT_REGISTRY}"' + - docker logout + - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}" + - make IMAGE_NAME=${OUT_IMAGE_NAME} VERSION=${OUT_IMAGE_VERSION} -f build/container/Makefile push-${DIST} + +# Define a staging release step that pushes an image to an internal "staging" repository +# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps +# outside of the release process. +.release:staging: + extends: + - .release + variables: + OUT_REGISTRY_USER: "${CI_REGISTRY_USER}" + OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}" + OUT_REGISTRY: "${CI_REGISTRY}" + OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/staging/container-toolkit" + +# Define an external release step that pushes an image to an external repository. +# This includes a devlopment image off master. +.release:external: + extends: + - .release + rules: + - if: $CI_COMMIT_TAG + variables: + OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}" + - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH + variables: + OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}" + +# Define the release jobs +release:staging-centos7: + extends: + - .release:staging + - .dist-centos7 + needs: + - image-centos7 + +release:staging-centos8: + extends: + - .release:staging + - .dist-centos8 + needs: + - image-centos8 + +release:staging-ubi8: + extends: + - .release:staging + - .dist-ubi8 + needs: + - image-ubi8 + +release:staging-ubuntu18.04: + extends: + - .release:staging + - .dist-ubuntu18.04 + needs: + - test-toolkit-ubuntu18.04 + - test-containerd-ubuntu18.04 + - test-crio-ubuntu18.04 + - test-docker-ubuntu18.04 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 423f3782..3542d9b5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,40 +15,18 @@ include: - .common-ci.yml -.build-setup: - before_script: - - apk update - - apk upgrade - - apk add coreutils build-base sed git bash make - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes - -# build-one jobs build packages for a single OS / ARCH combination. -# -# They are run during the first stage of the pipeline as a smoke test to ensure -# that we can successfully build packages on all of our architectures for a -# single OS. They are triggered on any change to an MR. No artifacts are -# produced as part of build-one jobs. -.build-one-setup: - extends: - - .build-setup - stage: build - rules: - - if: $CI_MERGE_REQUEST_ID - # build-all jobs build packages for every OS / ARCH combination we support. # # They are run under two conditions: # 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0) # 2) Manually by a reviewer just before merging a MR. -# -# Unlike build-one jobs, it takes a long time to build the full suite -# OS / ARCH combinations, so this is optimized to only run once per MR -# (assuming it all passes). A full set of artifacts including the packages -# built for each OS / ARCH are produced as a result of these jobs. -.build-all-setup: +.build-all-for-arch: + variables: + # Setting DIST=docker invokes the docker- release targets + DIST: docker extends: - - .build-setup - stage: build-long + - .package-build + stage: build-all timeout: 2h 30m rules: - if: $CI_COMMIT_TAG @@ -56,66 +34,29 @@ include: - if: $CI_MERGE_REQUEST_ID when: always - variables: - ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}" - ARTIFACTS_DIR: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}" - DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_DIR}" - - artifacts: - name: ${ARTIFACTS_NAME} - paths: - - ${ARTIFACTS_DIR} - -# The full set of build-one jobs organizes to build -# ubuntu18.04 in parallel on each of our supported ARCHs. -build-one-amd64: - extends: - - .build-one-setup - script: - - make ubuntu18.04-amd64 - rules: - - when: always - -build-one-ppc64le: - extends: - - .build-one-setup - script: - - make ubuntu18.04-ppc64le - -build-one-arm64: - extends: - - .build-one-setup - script: - - make ubuntu18.04-arm64 - # The full set of build-all jobs organized to # have builds for each ARCH run in parallel. build-all-amd64: extends: - - .build-all-setup - script: - - make docker-amd64 + - .build-all-for-arch + - .arch-amd64 build-all-x86_64: extends: - - .build-all-setup - script: - - make docker-x86_64 + - .build-all-for-arch + - .arch-x86_64 build-all-ppc64le: extends: - - .build-all-setup - script: - - make docker-ppc64le + - .build-all-for-arch + - .arch-ppc64le build-all-arm64: extends: - - .build-all-setup - script: - - make docker-arm64 + - .build-all-for-arch + - .arch-arm64 build-all-aarch64: extends: - - .build-all-setup - script: - - make docker-aarch64 + - .build-all-for-arch + - .arch-aarch64 diff --git a/build/container/Dockerfile.centos b/build/container/Dockerfile.centos index 197343af..b47778ec 100644 --- a/build/container/Dockerfile.centos +++ b/build/container/Dockerfile.centos @@ -29,14 +29,15 @@ ARG GOPATH=/artifacts # Install the experiemental nvidia-container-runtime # NOTE: This will be integrated into the nvidia-container-toolkit package / repo ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental -RUN go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION} +RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION} +WORKDIR /build COPY . . # NOTE: Until the config utilities are properly integrated into the # nvidia-container-toolkit repository, these are built from the `tools` folder # and not `cmd`. -RUN go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/... +RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/... FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} @@ -48,7 +49,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES=utility WORKDIR /artifacts/packages ARG ARTIFACTS_DIR -COPY ${ARTIFACTS_DIR}/* /artifacts/packages +COPY ${ARTIFACTS_DIR}/* /artifacts/packages/ ARG PACKAGE_VERSION RUN yum localinstall -y \ diff --git a/build/container/Dockerfile.ubuntu b/build/container/Dockerfile.ubuntu index e0ddd5cd..6cd3bfa1 100644 --- a/build/container/Dockerfile.ubuntu +++ b/build/container/Dockerfile.ubuntu @@ -29,14 +29,15 @@ ARG GOPATH=/artifacts # Install the experiemental nvidia-container-runtime # NOTE: This will be integrated into the nvidia-container-toolkit package / repo ARG NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION=experimental -RUN go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION} +RUN GOPATH=/artifacts go install github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-container-runtime.experimental@${NVIDIA_CONTAINER_RUNTIME_EXPERIMENTAL_VERSION} +WORKDIR /build COPY . . # NOTE: Until the config utilities are properly integrated into the # nvidia-container-toolkit repository, these are built from the `tools` folder # and not `cmd`. -RUN go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/... +RUN GOPATH=/artifacts go install -ldflags="-s -w -X 'main.Version=${VERSION}'" ./tools/... FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} @@ -54,7 +55,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES=utility WORKDIR /artifacts/packages ARG ARTIFACTS_DIR -COPY ${ARTIFACTS_DIR}/* /artifacts/packages +COPY ${ARTIFACTS_DIR}/* /artifacts/packages/ ARG PACKAGE_VERSION RUN dpkg -i \ @@ -64,7 +65,7 @@ RUN dpkg -i \ WORKDIR /work -COPY --from=build /artifacts/bin /work +COPY --from=build /artifacts/bin /work/ ENV PATH=/work:$PATH diff --git a/build/container/Makefile b/build/container/Makefile index ca7ad7c2..6fc7e6dc 100644 --- a/build/container/Makefile +++ b/build/container/Makefile @@ -40,7 +40,7 @@ DEFAULT_PUSH_TARGET := ubuntu18.04 TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8 BUILD_TARGETS := $(patsubst %, build-%, $(TARGETS)) -# PUSH_TARGETS := $(patsubst %, push-%, $(TARGETS)) +PUSH_TARGETS := $(patsubst %, push-%, $(TARGETS)) TEST_TARGETS := $(patsubst %, test-%, $(TARGETS)) .PHONY: $(TARGETS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) @@ -76,8 +76,10 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_DIR) $(CURDIR) +ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR)) + build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu -build-ubuntu%: ARTIFACTS_DIR = dist/$(*)/amd64 +build-ubuntu%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/amd64 build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG)) build-ubuntu18.04: BASE_DIST := ubuntu18.04 @@ -85,12 +87,12 @@ build-ubuntu20.04: BASE_DIST := ubuntu20.04 build-ubi8: DOCKERFILE_SUFFIX := centos # TODO: Update this to use the centos8 packages -build-ubi8: ARTIFACTS_DIR = dist/centos7/x86_64 +build-ubi8: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/centos7/x86_64 build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1) build-ubi8: BASE_DIST := ubi8 build-centos%: DOCKERFILE_SUFFIX := centos -build-centos%: ARTIFACTS_DIR = dist/$(*)/x86_64 +build-centos%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/x86_64 build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1) build-centos7: BASE_DIST := centos7