From 97762ce5f9d8c90e89fedccde91f92a6a66cd0c1 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 18 Feb 2022 14:45:38 +0200 Subject: [PATCH 01/12] Update submodules Signed-off-by: Evan Lezar --- third_party/libnvidia-container | 2 +- third_party/nvidia-container-runtime | 2 +- third_party/nvidia-docker | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/third_party/libnvidia-container b/third_party/libnvidia-container index abd4e14d..88f28f41 160000 --- a/third_party/libnvidia-container +++ b/third_party/libnvidia-container @@ -1 +1 @@ -Subproject commit abd4e14d8cb923e2a70b7dcfee55fbc16bffa353 +Subproject commit 88f28f41fd20018d4244df5132d3d6565d6bbb7f diff --git a/third_party/nvidia-container-runtime b/third_party/nvidia-container-runtime index 876bafab..cb2278e9 160000 --- a/third_party/nvidia-container-runtime +++ b/third_party/nvidia-container-runtime @@ -1 +1 @@ -Subproject commit 876bafab858eda94867e1c42053881bd28328288 +Subproject commit cb2278e9d320034c05b949ba8c784f8e4e81d7c9 diff --git a/third_party/nvidia-docker b/third_party/nvidia-docker index 614bb9be..4175b0fc 160000 --- a/third_party/nvidia-docker +++ b/third_party/nvidia-docker @@ -1 +1 @@ -Subproject commit 614bb9be41f847000d6a15126f95964c163e1cb6 +Subproject commit 4175b0fc355348dc85415bccef80684f42b20662 From ea4013fcd5ae987fec83b5862de458d7278a92a8 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 18 Feb 2022 12:50:37 +0200 Subject: [PATCH 02/12] Fix centos8 builds Signed-off-by: Evan Lezar --- build/container/Dockerfile.centos | 5 +++-- docker/Dockerfile.centos | 5 +++-- test/release/docker/centos8/Dockerfile | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/build/container/Dockerfile.centos b/build/container/Dockerfile.centos index 1b580b00..5358ea68 100644 --- a/build/container/Dockerfile.centos +++ b/build/container/Dockerfile.centos @@ -44,11 +44,12 @@ FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} ARG BASE_DIST # See https://www.centos.org/centos-linux-eol/ -# and https://stackoverflow.com/a/70930049 +# and https://stackoverflow.com/a/70930049 for move to vault.centos.org +# and https://serverfault.com/questions/1093922/failing-to-run-yum-update-in-centos-8 for move to vault.epel.cloud RUN [[ "${BASE_DIST}" != "centos8" ]] || \ ( \ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \ - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* \ + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* \ ) ENV NVIDIA_DISABLE_REQUIRE="true" diff --git a/docker/Dockerfile.centos b/docker/Dockerfile.centos index 71cdb761..d7a7d21d 100644 --- a/docker/Dockerfile.centos +++ b/docker/Dockerfile.centos @@ -3,11 +3,12 @@ FROM ${BASEIMAGE} ARG BASEIMAGE # See https://www.centos.org/centos-linux-eol/ -# and https://stackoverflow.com/a/70930049 +# and https://stackoverflow.com/a/70930049 for move to vault.centos.org +# and https://serverfault.com/questions/1093922/failing-to-run-yum-update-in-centos-8 for move to vault.epel.cloud RUN [[ "${BASEIMAGE}" != "centos:8" ]] || \ ( \ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \ - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* \ + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* \ ) RUN yum install -y \ ca-certificates \ diff --git a/test/release/docker/centos8/Dockerfile b/test/release/docker/centos8/Dockerfile index 20cc44f8..185cb6e7 100644 --- a/test/release/docker/centos8/Dockerfile +++ b/test/release/docker/centos8/Dockerfile @@ -3,11 +3,12 @@ FROM ${BASEIMAGE} ARG BASEIMAGE # See https://www.centos.org/centos-linux-eol/ -# and https://stackoverflow.com/a/70930049 +# and https://stackoverflow.com/a/70930049 for move to vault.centos.org +# and https://serverfault.com/questions/1093922/failing-to-run-yum-update-in-centos-8 for move to vault.epel.cloud RUN [[ "${BASEIMAGE}" != "centos:8" ]] || \ ( \ sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* && \ - sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* \ + sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.epel.cloud|g' /etc/yum.repos.d/CentOS-Linux-* \ ) RUN yum install -y \ From 980185db55ea6d71ee7f291647e61f75f83d5d4e Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 18 Feb 2022 16:13:38 +0200 Subject: [PATCH 03/12] Remove unneeded build-all CI steps Signed-off-by: Evan Lezar --- .common-ci.yml | 1 - .gitlab-ci.yml | 42 ------------------------------------------ 2 files changed, 43 deletions(-) diff --git a/.common-ci.yml b/.common-ci.yml index 33e61894..525df58c 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -32,7 +32,6 @@ stages: - test - scan - release - - build-all # Define the distribution targets .dist-amazonlinux2: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 27f93709..4dc22b62 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -328,45 +328,3 @@ test-docker-ubuntu18.04: needs: - image-ubuntu18.04 -# build-all jobs build packages for every OS / ARCH combination we support. -# -# They are run under two conditions: -# 1) Automatically whenever a new tag is pushed to the repo (e.g. v1.1.0) -# 2) Manually by a reviewer just before merging a MR. -.build-all-for-arch: - variables: - # Setting DIST=docker invokes the docker- release targets - DIST: docker - extends: - - .package-build - stage: build-all - rules: - - if: $CI_COMMIT_TAG - when: always - -# The full set of build-all jobs organized to -# have builds for each ARCH run in parallel. -build-all-amd64: - extends: - - .build-all-for-arch - - .arch-amd64 - -build-all-x86_64: - extends: - - .build-all-for-arch - - .arch-x86_64 - -build-all-ppc64le: - extends: - - .build-all-for-arch - - .arch-ppc64le - -build-all-arm64: - extends: - - .build-all-for-arch - - .arch-arm64 - -build-all-aarch64: - extends: - - .build-all-for-arch - - .arch-aarch64 From a1ce176fc461761ad0229a510daa2a62fe301ea3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 2 Feb 2022 13:50:24 +0100 Subject: [PATCH 04/12] Ensure that Ubuntu20.04 images also build Signed-off-by: Evan Lezar --- build/container/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/container/Makefile b/build/container/Makefile index 3bda4970..47bb01dd 100644 --- a/build/container/Makefile +++ b/build/container/Makefile @@ -83,7 +83,7 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT) build-ubuntu%: BASE_DIST = $(*) build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu build-ubuntu%: PACKAGE_ARCH := amd64 -build-ubuntu%: PACKAGE_DIST = $(BASE_DIST) +build-ubuntu%: PACKAGE_DIST = ubuntu18.04 build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG)) # TODO: Update this to use the centos8 packages From 74ddfe901a94a4ab841277474f5c710841a3dbf7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 1 Feb 2022 13:34:16 +0100 Subject: [PATCH 05/12] Specify docker platform args for build and run commands Signed-off-by: Evan Lezar --- docker/docker.mk | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/docker.mk b/docker/docker.mk index 3a33034c..9a29677a 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -122,6 +122,7 @@ docker-build-%: docker pull --platform=linux/$(ARCH) $(BASEIMAGE) DOCKER_BUILDKIT=1 \ $(DOCKER) build \ + --platform=linux/$(ARCH) \ --progress=plain \ --build-arg BASEIMAGE="$(BASEIMAGE)" \ --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \ @@ -131,6 +132,7 @@ docker-build-%: --tag $(BUILDIMAGE) \ --file $(DOCKERFILE) . $(DOCKER) run \ + --platform=linux/$(ARCH) \ -e DISTRIB \ -e SECTION \ -v $(ARTIFACTS_DIR):/dist \ From ec7de9c4e836a91afbd4d75aa7f075adaad4ea3f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 1 Feb 2022 11:10:07 +0100 Subject: [PATCH 06/12] Rename TARGETS make variable to DISTRIBUTIONS Signed-off-by: Evan Lezar --- build/container/Makefile | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/build/container/Makefile b/build/container/Makefile index 47bb01dd..ed76835a 100644 --- a/build/container/Makefile +++ b/build/container/Makefile @@ -31,15 +31,15 @@ IMAGE = $(IMAGE_NAME):$(IMAGE_TAG) ##### Public rules ##### DEFAULT_PUSH_TARGET := ubuntu18.04 -TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8 +DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8 META_TARGETS := packaging -BUILD_TARGETS := $(patsubst %,build-%,$(TARGETS) $(META_TARGETS)) -PUSH_TARGETS := $(patsubst %,push-%,$(TARGETS) $(META_TARGETS)) -TEST_TARGETS := $(patsubst %,test-%, $(TARGETS)) +BUILD_TARGETS := $(patsubst %,build-%,$(DISTRIBUTIONS) $(META_TARGETS)) +PUSH_TARGETS := $(patsubst %,push-%,$(DISTRIBUTIONS) $(META_TARGETS)) +TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS)) -.PHONY: $(TARGETS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) +.PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) push-%: DIST = $(*) $(PUSH_TARGETS): push-%: From e8d555f1555ad2aa54f2be94887a0dad232add75 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 1 Feb 2022 11:54:03 +0100 Subject: [PATCH 07/12] Allow buildx to be used for mulit-arch images This change allows for docker buildx to be used to build container images. This also allows multi-arch images being built. In addition to using docker buildx to build images, regctl as a replacement for the docker push command to release images. This tool also supports regctl. The selection of docker buildx (and regctl) is controlled by a BUILD_MULTI_ARCH_IMAGES make variable. If this is 'true', the build-% make targets for the toolkit container will be run through buildx and the equivalent push-% targets will trigger a regctl command. Signed-off-by: Evan Lezar --- build/container/Dockerfile.centos | 6 ++++-- build/container/Dockerfile.ubuntu | 13 +++++++++++- build/container/Makefile | 35 ++++++++++++++++++++++++------- versions.mk | 3 +++ 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/build/container/Dockerfile.centos b/build/container/Dockerfile.centos index 5358ea68..3a4c2d6a 100644 --- a/build/container/Dockerfile.centos +++ b/build/container/Dockerfile.centos @@ -63,8 +63,10 @@ COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST} WORKDIR /artifacts/packages ARG PACKAGE_VERSION -ARG PACKAGE_ARCH -RUN yum localinstall -y \ +ARG TARGETARCH +ENV PACKAGE_ARCH ${TARGETARCH} +RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \ + yum localinstall -y \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm diff --git a/build/container/Dockerfile.ubuntu b/build/container/Dockerfile.ubuntu index ff3d30bd..b912d740 100644 --- a/build/container/Dockerfile.ubuntu +++ b/build/container/Dockerfile.ubuntu @@ -45,6 +45,7 @@ FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ libcap2 \ + curl \ && \ rm -rf /var/lib/apt/lists/* @@ -59,7 +60,17 @@ COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST} WORKDIR /artifacts/packages ARG PACKAGE_VERSION -ARG PACKAGE_ARCH +ARG TARGETARCH +ENV PACKAGE_ARCH ${TARGETARCH} + +ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container" +ARG LIBNVIDIA_CONTAINER0_VERSION +RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \ + curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \ + --output ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb && \ + dpkg -i ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb; \ + fi + RUN dpkg -i \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \ diff --git a/build/container/Makefile b/build/container/Makefile index ed76835a..a131ef55 100644 --- a/build/container/Makefile +++ b/build/container/Makefile @@ -12,7 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -DOCKER ?= docker +BUILD_MULTI_ARCH_IMAGES ?= false +DOCKER ?= docker + +BUILDX = +ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) +BUILDX = buildx +endif + MKDIR ?= mkdir DIST_DIR ?= $(CURDIR)/dist @@ -42,6 +49,12 @@ TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS)) .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) push-%: DIST = $(*) +ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) +# For multi-arch images we use buildx and set push=true, invoking the build +# target directly +push-%: DOCKER_BUILD_OPTIONS = --output=type=image,push=true +$(PUSH_TARGETS): push-%: build-% +else $(PUSH_TARGETS): push-%: $(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)" @@ -54,26 +67,34 @@ endif push-short: $(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(IMAGE_NAME):$(VERSION)" $(DOCKER) push "$(IMAGE_NAME):$(VERSION)" - +endif build-%: DIST = $(*) build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX) ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR)) +# TODO: This should be distribution dependand +DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 + +ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) +DOCKER_BUILD_OPTIONS = --output=type=image,push=false +endif + # Use a generic build target to build the relevant images $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT) DOCKER_BUILDKIT=1 \ $(DOCKER) build --pull \ - --platform=linux/amd64 \ + $(DOCKER_BUILD_OPTIONS) \ + $(DOCKER_BUILD_PLATFORM_OPTIONS) \ --tag $(IMAGE) \ --build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \ --build-arg BASE_DIST="$(BASE_DIST)" \ --build-arg CUDA_VERSION="$(CUDA_VERSION)" \ --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \ + --build-arg LIBNVIDIA_CONTAINER0_VERSION="$(LIBNVIDIA_CONTAINER0_DEPENDENCY)" \ --build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \ --build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \ - --build-arg PACKAGE_ARCH="$(PACKAGE_ARCH)" \ --build-arg VERSION="$(VERSION)" \ --build-arg CVE_UPDATES="$(CVE_UPDATES)" \ -f $(DOCKERFILE) \ @@ -82,20 +103,18 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT) build-ubuntu%: BASE_DIST = $(*) build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu -build-ubuntu%: PACKAGE_ARCH := amd64 build-ubuntu%: PACKAGE_DIST = ubuntu18.04 build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG)) +build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION) # TODO: Update this to use the centos8 packages build-ubi8: BASE_DIST := ubi8 build-ubi8: DOCKERFILE_SUFFIX := centos -build-ubi8: PACKAGE_ARCH := x86_64 build-ubi8: PACKAGE_DIST = centos7 build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1) build-centos%: BASE_DIST = $(*) build-centos%: DOCKERFILE_SUFFIX := centos -build-centos%: PACKAGE_ARCH := x86_64 build-centos%: PACKAGE_DIST = $(BASE_DIST) build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1) @@ -104,6 +123,8 @@ build-packaging: DOCKERFILE_SUFFIX := packaging build-packaging: PACKAGE_ARCH := amd64 build-packaging: PACKAGE_DIST = all build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG)) +# We only generate a single image for packaging targets +build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 # Test targets test-%: DIST = $(*) diff --git a/versions.mk b/versions.mk index 9219b90b..8113a619 100644 --- a/versions.mk +++ b/versions.mk @@ -21,5 +21,8 @@ LIB_TAG := rc.1 NVIDIA_DOCKER_VERSION := 2.10.0 NVIDIA_CONTAINER_RUNTIME_VERSION := 3.9.0 +# Specify the expected libnvidia-container0 version for arm64-based ubuntu builds. +LIBNVIDIA_CONTAINER0_VERSION := 0.10.0+jetpack + CUDA_VERSION := 11.6.0 GOLANG_VERSION := 1.16.4 From a0d2b22a54890078a9a974ea07788cda18624dac Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 1 Feb 2022 14:14:33 +0100 Subject: [PATCH 08/12] Enable multi-arch builds This change adds arm64/aarch64 images to supported distributions. This is triggered if BUILD_MULTI_ARCH_IMAGE=true. Note that for ubi8 images this means that we switch to using centos8 packages instead of centos7 since we do not build aarch64 packages for the latter. This also means that for centos7 we only build x86_64 images. Signed-off-by: Evan Lezar --- .gitlab-ci.yml | 7 +++--- build/container/Makefile | 39 +++++++++++++--------------------- build/container/multi-arch.mk | 34 +++++++++++++++++++++++++++++ build/container/native-only.mk | 22 +++++++++++++++++++ 4 files changed, 75 insertions(+), 27 deletions(-) create mode 100644 build/container/multi-arch.mk create mode 100644 build/container/native-only.mk diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 4dc22b62..779824a7 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -233,9 +233,10 @@ image-ubi8: - .package-artifacts - .dist-ubi8 needs: - # Note: The ubi8 image currently uses the centos7 packages - - package-centos7-ppc64le - - package-centos7-x86_64 + # Note: The ubi8 image uses the centos8 packages + - package-centos8-aarch64 + - package-centos8-x86_64 + - package-centos8-ppc64le image-ubuntu18.04: extends: diff --git a/build/container/Makefile b/build/container/Makefile index a131ef55..2523a577 100644 --- a/build/container/Makefile +++ b/build/container/Makefile @@ -32,10 +32,16 @@ IMAGE_NAME := $(REGISTRY)/container-toolkit endif VERSION ?= $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG)) +IMAGE_VERSION := $(VERSION) IMAGE_TAG ?= $(VERSION)-$(DIST) IMAGE = $(IMAGE_NAME):$(IMAGE_TAG) +OUT_IMAGE_NAME ?= $(IMAGE_NAME) +OUT_IMAGE_VERSION ?= $(IMAGE_VERSION) +OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST) +OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG) + ##### Public rules ##### DEFAULT_PUSH_TARGET := ubuntu18.04 DISTRIBUTIONS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8 @@ -48,15 +54,11 @@ TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS)) .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) -push-%: DIST = $(*) -ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) -# For multi-arch images we use buildx and set push=true, invoking the build -# target directly -push-%: DOCKER_BUILD_OPTIONS = --output=type=image,push=true -$(PUSH_TARGETS): push-%: build-% +ifneq ($(BUILD_MULTI_ARCH_IMAGES),true) +include $(CURDIR)/build/container/native-only.mk else -$(PUSH_TARGETS): push-%: - $(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)" +include $(CURDIR)/build/container/multi-arch.mk +endif # For the default push target we also push a short tag equal to the version. # We skip this for the development release @@ -64,27 +66,19 @@ DEVEL_RELEASE_IMAGE_VERSION ?= devel ifneq ($(strip $(VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION)) push-$(DEFAULT_PUSH_TARGET): push-short endif -push-short: - $(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(IMAGE_NAME):$(VERSION)" - $(DOCKER) push "$(IMAGE_NAME):$(VERSION)" -endif + +push-%: DIST = $(*) +push-short: DIST = $(DEFAULT_PUSH_TARGET) build-%: DIST = $(*) build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX) ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR)) -# TODO: This should be distribution dependand -DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 - -ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) -DOCKER_BUILD_OPTIONS = --output=type=image,push=false -endif - # Use a generic build target to build the relevant images $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT) DOCKER_BUILDKIT=1 \ - $(DOCKER) build --pull \ + $(DOCKER) $(BUILDX) build --pull \ $(DOCKER_BUILD_OPTIONS) \ $(DOCKER_BUILD_PLATFORM_OPTIONS) \ --tag $(IMAGE) \ @@ -107,10 +101,9 @@ build-ubuntu%: PACKAGE_DIST = ubuntu18.04 build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG)) build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION) -# TODO: Update this to use the centos8 packages build-ubi8: BASE_DIST := ubi8 build-ubi8: DOCKERFILE_SUFFIX := centos -build-ubi8: PACKAGE_DIST = centos7 +build-ubi8: PACKAGE_DIST = centos8 build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1) build-centos%: BASE_DIST = $(*) @@ -123,8 +116,6 @@ build-packaging: DOCKERFILE_SUFFIX := packaging build-packaging: PACKAGE_ARCH := amd64 build-packaging: PACKAGE_DIST = all build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG)) -# We only generate a single image for packaging targets -build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 # Test targets test-%: DIST = $(*) diff --git a/build/container/multi-arch.mk b/build/container/multi-arch.mk new file mode 100644 index 00000000..94a3527c --- /dev/null +++ b/build/container/multi-arch.mk @@ -0,0 +1,34 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +PUSH_ON_BUILD ?= false +DOCKER_BUILD_OPTIONS = --output=type=image,push=$(PUSH_ON_BUILD) +DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64,linux/arm64 + +REGCTL ?= regctl +$(PUSH_TARGETS): push-%: + $(REGCTL) \ + image copy \ + $(IMAGE) $(OUT_IMAGE) + +push-short: + $(REGCTL) \ + image copy \ + $(IMAGE) $(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION) + +# We only have x86_64 packages for centos7 +build-centos7: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 + +# We only generate a single image for packaging targets +build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 diff --git a/build/container/native-only.mk b/build/container/native-only.mk new file mode 100644 index 00000000..be508588 --- /dev/null +++ b/build/container/native-only.mk @@ -0,0 +1,22 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 + +$(PUSH_TARGETS): push-%: + $(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)" + +push-short: + $(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(IMAGE_NAME):$(VERSION)" + $(DOCKER) push "$(IMAGE_NAME):$(VERSION)" From b05db2befe77910245b91126c06ee4a49a6f3050 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 1 Feb 2022 14:17:47 +0100 Subject: [PATCH 09/12] Enable multi-arch builds in CI Signed-off-by: Evan Lezar --- .common-ci.yml | 33 ++++++++++++++++++++++----------- .gitlab-ci.yml | 16 +++++++++++++++- .nvidia-ci.yml | 10 +++++----- 3 files changed, 42 insertions(+), 17 deletions(-) diff --git a/.common-ci.yml b/.common-ci.yml index 525df58c..5b270268 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -20,6 +20,7 @@ default: variables: GIT_SUBMODULE_STRATEGY: recursive BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}" + BUILD_MULTI_ARCH_IMAGES: "true" stages: - image @@ -117,20 +118,30 @@ test-packaging: needs: - image-packaging +# Download the regctl binary for use in the release steps +.regctl-setup: + before_script: + - export REGCTL_VERSION=v0.3.10 + - apk add --no-cache curl + - mkdir -p bin + - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64 + - chmod a+x bin/regctl + - export PATH=$(pwd)/bin:${PATH} + # .release forms the base of the deployment jobs which push images to the CI registry. # This is extended with the version to be deployed (e.g. the SHA or TAG) and the # target os. .release: - stage: - release + stage: release variables: # Define the source image for the release IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" VERSION: "${CI_COMMIT_SHORT_SHA}" # OUT_IMAGE_VERSION is overridden for external releases OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}" - stage: release before_script: + - !reference [.regctl-setup, before_script] + # We ensure that the OUT_IMAGE_VERSION is set - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1' @@ -138,16 +149,16 @@ test-packaging: # need to tag the image. # Note: a leading 'v' is stripped from the version if present - apk add --no-cache make bash - - 'echo "Logging in to CI registry ${CI_REGISTRY}"' - - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" - - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}" script: - - docker tag "${IMAGE_NAME}:${VERSION}-${DIST}" "${OUT_IMAGE_NAME}:${OUT_IMAGE_VERSION}-${DIST}" # Log in to the "output" registry, tag the image and push the image - - 'echo "Logging in to output registry ${OUT_REGISTRY}"' - - docker logout - - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}" - - make IMAGE_NAME=${OUT_IMAGE_NAME} VERSION=${OUT_IMAGE_VERSION} -f build/container/Makefile push-${DIST} + - 'echo "Logging in to CI registry ${CI_REGISTRY}"' + - regctl registry login "${CI_REGISTRY}" -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" + - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || echo "Logging in to output registry ${OUT_REGISTRY}"' + - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"' + + # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the + # Target + - make -f build/container/Makefile push-${DIST} # Define a staging release step that pushes an image to an internal "staging" repository # This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 779824a7..522b2399 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -194,19 +194,33 @@ package-ubuntu18.04-ppc64le: - .dist-ubuntu18.04 - .arch-ppc64le +.buildx-setup: + before_script: + - export BUILDX_VERSION=v0.6.3 + - apk add --no-cache curl + - mkdir -p ~/.docker/cli-plugins + - curl -sSLo ~/.docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64" + - chmod a+x ~/.docker/cli-plugins/docker-buildx + + - docker buildx create --use --platform=linux/amd64,linux/arm64 + + - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes' + # Define the image build targets .image-build: stage: image-build variables: IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" VERSION: "${CI_COMMIT_SHORT_SHA}" + PUSH_ON_BUILD: "true" before_script: + - !reference [.buildx-setup, before_script] + - apk add --no-cache bash make - 'echo "Logging in to CI registry ${CI_REGISTRY}"' - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" script: - make -f build/container/Makefile build-${DIST} - - make -f build/container/Makefile push-${DIST} image-centos7: extends: diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index bf926571..478526e9 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -56,13 +56,13 @@ variables: - job_execution_timeout - stuck_or_timeout_failure before_script: + - !reference [.regctl-setup, before_script] + - apk add --no-cache make bash - > - docker pull ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity ) + regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity ) script: - - docker pull ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} - - docker tag ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} ${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} - - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}" - - docker push ${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} + - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" + - make -f build/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST} image-centos7: extends: From cc593087d2962d426da2dee3001c25c7fb9b5900 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 8 Feb 2022 11:36:57 +0100 Subject: [PATCH 10/12] Also search /usr/lib/aarch64-linux-gnu for libnvidia-container libs Signed-off-by: Evan Lezar --- tools/container/toolkit/toolkit.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tools/container/toolkit/toolkit.go b/tools/container/toolkit/toolkit.go index bcc8f805..273fcc90 100644 --- a/tools/container/toolkit/toolkit.go +++ b/tools/container/toolkit/toolkit.go @@ -422,6 +422,7 @@ func findLibrary(root string, libName string) (string, error) { candidateDirs := []string{ "/usr/lib64", "/usr/lib/x86_64-linux-gnu", + "/usr/lib/aarch64-linux-gnu", } for _, d := range candidateDirs { From 93ca91ac3f4309403a09596e256b133e84808400 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 18 Feb 2022 14:50:40 +0200 Subject: [PATCH 11/12] Add multi-arch image scans Signed-off-by: Evan Lezar --- .common-ci.yml | 9 +++++++++ .nvidia-ci.yml | 50 +++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 54 insertions(+), 5 deletions(-) diff --git a/.common-ci.yml b/.common-ci.yml index 5b270268..de6e9863 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -97,6 +97,15 @@ stages: variables: ARCH: x86_64 +# Define the platform targets +.platform-amd64: + variables: + PLATFORM: linux/amd64 + +.platform-arm64: + variables: + PLATFORM: linux/arm64 + # Define test helpers .integration: stage: test diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index 478526e9..d7af6a19 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -112,7 +112,7 @@ image-packaging: before_script: - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" # TODO: We should specify the architecture here and scan all architectures - - docker pull "${IMAGE}" + - docker pull --platform="${PLATFORM}" "${IMAGE}" - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}" - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0) - > @@ -131,34 +131,74 @@ image-packaging: - policy_evaluation.json # Define the scan targets -scan-centos7: +scan-centos7-amd64: extends: - .scan - .dist-centos7 + - .platform-amd64 needs: - image-centos7 -scan-centos8: +scan-centos7-arm64: + extends: + - .scan + - .dist-centos7 + - .platform-arm64 + needs: + - image-centos7 + - scan-centos7-amd64 + +scan-centos8-amd64: extends: - .scan - .dist-centos8 + - .platform-amd64 needs: - image-centos8 -scan-ubuntu18.04: +scan-centos8-arm64: + extends: + - .scan + - .dist-centos8 + - .platform-arm64 + needs: + - image-centos8 + - scan-centos8-amd64 + +scan-ubuntu18.04-amd64: extends: - .scan - .dist-ubuntu18.04 + - .platform-amd64 needs: - image-ubuntu18.04 -scan-ubi8: +scan-ubuntu18.04-arm64: + extends: + - .scan + - .dist-ubuntu18.04 + - .platform-arm64 + needs: + - image-ubuntu18.04 + - scan-ubuntu18.04-amd64 + +scan-ubi8-amd64: extends: - .scan - .dist-ubi8 + - .platform-amd64 needs: - image-ubi8 +scan-ubi8-arm64: + extends: + - .scan + - .dist-ubi8 + - .platform-arm64 + needs: + - image-ubi8 + - scan-ubi8-amd64 + # Define external release helpers .release:ngc: extends: From 2e4ed47ac46ae617b12aefcc65941d9d6ecd6db3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 22 Feb 2022 10:18:01 +0200 Subject: [PATCH 12/12] Fix pushing of short tag for devel images Signed-off-by: Evan Lezar --- .nvidia-ci.yml | 1 + build/container/Makefile | 6 +++++- build/container/native-only.mk | 7 ++++--- 3 files changed, 10 insertions(+), 4 deletions(-) diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index d7af6a19..87ff1445 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -46,6 +46,7 @@ variables: OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}" OUT_REGISTRY: "${CI_REGISTRY}" OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" + PUSH_MULTIPLE_TAGS: "false" # We delay the job start to allow the public pipeline to generate the required images. when: delayed start_in: 30 minutes diff --git a/build/container/Makefile b/build/container/Makefile index 2523a577..7a1186e7 100644 --- a/build/container/Makefile +++ b/build/container/Makefile @@ -63,7 +63,11 @@ endif # For the default push target we also push a short tag equal to the version. # We skip this for the development release DEVEL_RELEASE_IMAGE_VERSION ?= devel -ifneq ($(strip $(VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION)) +PUSH_MULTIPLE_TAGS ?= true +ifeq ($(strip $(OUT_IMAGE_VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION)) +PUSH_MULTIPLE_TAGS = false +endif +ifeq ($(PUSH_MULTIPLE_TAGS),true) push-$(DEFAULT_PUSH_TARGET): push-short endif diff --git a/build/container/native-only.mk b/build/container/native-only.mk index be508588..aacde563 100644 --- a/build/container/native-only.mk +++ b/build/container/native-only.mk @@ -15,8 +15,9 @@ DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 $(PUSH_TARGETS): push-%: - $(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)" + $(DOCKER) tag "$(IMAGE)" "$(OUT_IMAGE)" + $(DOCKER) push "$(OUT_IMAGE)" push-short: - $(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(IMAGE_NAME):$(VERSION)" - $(DOCKER) push "$(IMAGE_NAME):$(VERSION)" + $(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)" + $(DOCKER) push "$(OUT_IMAGE_NAME):$(OUT_IMAGE_VERSION)"