From e8d555f1555ad2aa54f2be94887a0dad232add75 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 1 Feb 2022 11:54:03 +0100 Subject: [PATCH] Allow buildx to be used for mulit-arch images This change allows for docker buildx to be used to build container images. This also allows multi-arch images being built. In addition to using docker buildx to build images, regctl as a replacement for the docker push command to release images. This tool also supports regctl. The selection of docker buildx (and regctl) is controlled by a BUILD_MULTI_ARCH_IMAGES make variable. If this is 'true', the build-% make targets for the toolkit container will be run through buildx and the equivalent push-% targets will trigger a regctl command. Signed-off-by: Evan Lezar --- build/container/Dockerfile.centos | 6 ++++-- build/container/Dockerfile.ubuntu | 13 +++++++++++- build/container/Makefile | 35 ++++++++++++++++++++++++------- versions.mk | 3 +++ 4 files changed, 47 insertions(+), 10 deletions(-) diff --git a/build/container/Dockerfile.centos b/build/container/Dockerfile.centos index 5358ea68..3a4c2d6a 100644 --- a/build/container/Dockerfile.centos +++ b/build/container/Dockerfile.centos @@ -63,8 +63,10 @@ COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST} WORKDIR /artifacts/packages ARG PACKAGE_VERSION -ARG PACKAGE_ARCH -RUN yum localinstall -y \ +ARG TARGETARCH +ENV PACKAGE_ARCH ${TARGETARCH} +RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \ + yum localinstall -y \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm diff --git a/build/container/Dockerfile.ubuntu b/build/container/Dockerfile.ubuntu index ff3d30bd..b912d740 100644 --- a/build/container/Dockerfile.ubuntu +++ b/build/container/Dockerfile.ubuntu @@ -45,6 +45,7 @@ FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST} ARG DEBIAN_FRONTEND=noninteractive RUN apt-get update && apt-get install -y --no-install-recommends \ libcap2 \ + curl \ && \ rm -rf /var/lib/apt/lists/* @@ -59,7 +60,17 @@ COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST} WORKDIR /artifacts/packages ARG PACKAGE_VERSION -ARG PACKAGE_ARCH +ARG TARGETARCH +ENV PACKAGE_ARCH ${TARGETARCH} + +ARG LIBNVIDIA_CONTAINER_REPO="https://nvidia.github.io/libnvidia-container" +ARG LIBNVIDIA_CONTAINER0_VERSION +RUN if [ "${PACKAGE_ARCH}" = "arm64" ]; then \ + curl -L ${LIBNVIDIA_CONTAINER_REPO}/${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb \ + --output ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb && \ + dpkg -i ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container0_${LIBNVIDIA_CONTAINER0_VERSION}_${PACKAGE_ARCH}.deb; \ + fi + RUN dpkg -i \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \ ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \ diff --git a/build/container/Makefile b/build/container/Makefile index ed76835a..a131ef55 100644 --- a/build/container/Makefile +++ b/build/container/Makefile @@ -12,7 +12,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -DOCKER ?= docker +BUILD_MULTI_ARCH_IMAGES ?= false +DOCKER ?= docker + +BUILDX = +ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) +BUILDX = buildx +endif + MKDIR ?= mkdir DIST_DIR ?= $(CURDIR)/dist @@ -42,6 +49,12 @@ TEST_TARGETS := $(patsubst %,test-%, $(DISTRIBUTIONS)) .PHONY: $(DISTRIBUTIONS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS) push-%: DIST = $(*) +ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) +# For multi-arch images we use buildx and set push=true, invoking the build +# target directly +push-%: DOCKER_BUILD_OPTIONS = --output=type=image,push=true +$(PUSH_TARGETS): push-%: build-% +else $(PUSH_TARGETS): push-%: $(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)" @@ -54,26 +67,34 @@ endif push-short: $(DOCKER) tag "$(IMAGE_NAME):$(VERSION)-$(DEFAULT_PUSH_TARGET)" "$(IMAGE_NAME):$(VERSION)" $(DOCKER) push "$(IMAGE_NAME):$(VERSION)" - +endif build-%: DIST = $(*) build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX) ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR)) +# TODO: This should be distribution dependand +DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 + +ifeq ($(BUILD_MULTI_ARCH_IMAGES),true) +DOCKER_BUILD_OPTIONS = --output=type=image,push=false +endif + # Use a generic build target to build the relevant images $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT) DOCKER_BUILDKIT=1 \ $(DOCKER) build --pull \ - --platform=linux/amd64 \ + $(DOCKER_BUILD_OPTIONS) \ + $(DOCKER_BUILD_PLATFORM_OPTIONS) \ --tag $(IMAGE) \ --build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \ --build-arg BASE_DIST="$(BASE_DIST)" \ --build-arg CUDA_VERSION="$(CUDA_VERSION)" \ --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \ + --build-arg LIBNVIDIA_CONTAINER0_VERSION="$(LIBNVIDIA_CONTAINER0_DEPENDENCY)" \ --build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \ --build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \ - --build-arg PACKAGE_ARCH="$(PACKAGE_ARCH)" \ --build-arg VERSION="$(VERSION)" \ --build-arg CVE_UPDATES="$(CVE_UPDATES)" \ -f $(DOCKERFILE) \ @@ -82,20 +103,18 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT) build-ubuntu%: BASE_DIST = $(*) build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu -build-ubuntu%: PACKAGE_ARCH := amd64 build-ubuntu%: PACKAGE_DIST = ubuntu18.04 build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG)) +build-ubuntu%: LIBNVIDIA_CONTAINER0_DEPENDENCY=$(LIBNVIDIA_CONTAINER0_VERSION) # TODO: Update this to use the centos8 packages build-ubi8: BASE_DIST := ubi8 build-ubi8: DOCKERFILE_SUFFIX := centos -build-ubi8: PACKAGE_ARCH := x86_64 build-ubi8: PACKAGE_DIST = centos7 build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1) build-centos%: BASE_DIST = $(*) build-centos%: DOCKERFILE_SUFFIX := centos -build-centos%: PACKAGE_ARCH := x86_64 build-centos%: PACKAGE_DIST = $(BASE_DIST) build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1) @@ -104,6 +123,8 @@ build-packaging: DOCKERFILE_SUFFIX := packaging build-packaging: PACKAGE_ARCH := amd64 build-packaging: PACKAGE_DIST = all build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG)) +# We only generate a single image for packaging targets +build-packaging: DOCKER_BUILD_PLATFORM_OPTIONS = --platform=linux/amd64 # Test targets test-%: DIST = $(*) diff --git a/versions.mk b/versions.mk index 9219b90b..8113a619 100644 --- a/versions.mk +++ b/versions.mk @@ -21,5 +21,8 @@ LIB_TAG := rc.1 NVIDIA_DOCKER_VERSION := 2.10.0 NVIDIA_CONTAINER_RUNTIME_VERSION := 3.9.0 +# Specify the expected libnvidia-container0 version for arm64-based ubuntu builds. +LIBNVIDIA_CONTAINER0_VERSION := 0.10.0+jetpack + CUDA_VERSION := 11.6.0 GOLANG_VERSION := 1.16.4