From bdfd123b9d9abf277d08817da62ab53f3c863ef3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 10 Aug 2022 14:17:33 +0200 Subject: [PATCH 1/8] Switch to single docker file yum-based rpm builds This reuses the docker file for yum-based rpm distros (centos, amazonlinux) instead of maintaining two files with the same contents. Signed-off-by: Evan Lezar --- docker/Dockerfile.amazonlinux | 76 ------------------- .../{Dockerfile.centos => Dockerfile.rpm-yum} | 16 ++++ docker/docker.mk | 7 +- 3 files changed, 21 insertions(+), 78 deletions(-) delete mode 100644 docker/Dockerfile.amazonlinux rename docker/{Dockerfile.centos => Dockerfile.rpm-yum} (76%) diff --git a/docker/Dockerfile.amazonlinux b/docker/Dockerfile.amazonlinux deleted file mode 100644 index 6fd00c1e..00000000 --- a/docker/Dockerfile.amazonlinux +++ /dev/null @@ -1,76 +0,0 @@ -ARG BASEIMAGE -FROM ${BASEIMAGE} - -RUN yum install -y \ - ca-certificates \ - gcc \ - wget \ - git \ - rpm-build \ - make && \ - rm -rf /var/cache/yum/* - -ARG GOLANG_VERSION=0.0.0 -RUN set -eux; \ - \ - arch="$(uname -m)"; \ - case "${arch##*-}" in \ - x86_64 | amd64) ARCH='amd64' ;; \ - ppc64el | ppc64le) ARCH='ppc64le' ;; \ - aarch64) ARCH='arm64' ;; \ - *) echo "unsupported architecture"; exit 1 ;; \ - esac; \ - wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \ - | tar -C /usr/local -xz - -ENV GOPATH /go -ENV PATH $GOPATH/bin:/usr/local/go/bin:$PATH - -# packaging -ARG PKG_NAME -ARG PKG_VERS -ARG PKG_REV - -ENV VERSION $PKG_VERS -ENV RELEASE $PKG_REV - -# output directory -ENV DIST_DIR=/tmp/nvidia-container-toolkit-$PKG_VERS/SOURCES -RUN mkdir -p $DIST_DIR /dist - -# nvidia-container-toolkit -WORKDIR $GOPATH/src/nvidia-container-toolkit -COPY . . - -ARG GIT_COMMIT -ENV GIT_COMMIT ${GIT_COMMIT} -RUN make PREFIX=${DIST_DIR} cmds - -ARG CONFIG_TOML_SUFFIX -ENV CONFIG_TOML_SUFFIX ${CONFIG_TOML_SUFFIX} -COPY config/config.toml.${CONFIG_TOML_SUFFIX} $DIST_DIR/config.toml - -# Hook for Project Atomic's fork of Docker: https://github.com/projectatomic/docker/tree/docker-1.13.1-rhel#add-dockerhooks-exec-custom-hooks-for-prestartpoststop-containerspatch -# This might not be useful on Amazon Linux, but it's simpler to keep the RHEL -# and Amazon Linux packages identical. -COPY oci-nvidia-hook $DIST_DIR/oci-nvidia-hook - -# Hook for libpod/CRI-O: https://github.com/containers/libpod/blob/v0.8.5/pkg/hooks/docs/oci-hooks.5.md -COPY oci-nvidia-hook.json $DIST_DIR/oci-nvidia-hook.json - -WORKDIR $DIST_DIR/.. -COPY packaging/rpm . - -ARG LIBNVIDIA_CONTAINER_TOOLS_VERSION -ENV LIBNVIDIA_CONTAINER_TOOLS_VERSION ${LIBNVIDIA_CONTAINER_TOOLS_VERSION} - -CMD arch=$(uname -m) && \ - rpmbuild --clean --target=$arch -bb \ - -D "_topdir $PWD" \ - -D "release_date $(date +'%a %b %d %Y')" \ - -D "git_commit ${GIT_COMMIT}" \ - -D "version $VERSION" \ - -D "libnvidia_container_tools_version ${LIBNVIDIA_CONTAINER_TOOLS_VERSION}" \ - -D "release $RELEASE" \ - SPECS/nvidia-container-toolkit.spec && \ - mv RPMS/$arch/*.rpm /dist diff --git a/docker/Dockerfile.centos b/docker/Dockerfile.rpm-yum similarity index 76% rename from docker/Dockerfile.centos rename to docker/Dockerfile.rpm-yum index a64409d9..b4c2ed2f 100644 --- a/docker/Dockerfile.centos +++ b/docker/Dockerfile.rpm-yum @@ -1,3 +1,19 @@ +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is the dockerfile for building packages on yum-based RPM systems. + ARG BASEIMAGE FROM ${BASEIMAGE} diff --git a/docker/docker.mk b/docker/docker.mk index cf511afa..c21aa51b 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -14,10 +14,10 @@ # Supported OSs by architecture AMD64_TARGETS := ubuntu20.04 ubuntu18.04 ubuntu16.04 debian10 debian9 -X86_64_TARGETS := centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1 +X86_64_TARGETS := fedora35 centos7 centos8 rhel7 rhel8 amazonlinux2 opensuse-leap15.1 PPC64LE_TARGETS := ubuntu18.04 ubuntu16.04 centos7 centos8 rhel7 rhel8 ARM64_TARGETS := ubuntu20.04 ubuntu18.04 -AARCH64_TARGETS := centos8 rhel8 amazonlinux2 +AARCH64_TARGETS := fedora35 centos8 rhel8 amazonlinux2 # Define top-level build targets docker%: SHELL:=/bin/bash @@ -104,12 +104,14 @@ LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG) --centos%: OS := centos --centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1) --centos%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1) +--centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum --centos8%: BASEIMAGE = quay.io/centos/centos:stream8 # private amazonlinux target --amazonlinux%: OS := amazonlinux --amazonlinux%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1) --amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1) +--amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum # private opensuse-leap target --opensuse-leap%: OS = opensuse-leap @@ -123,6 +125,7 @@ LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG) --rhel%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1) --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM)) --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH) +--rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum --rhel8%: BASEIMAGE = quay.io/centos/centos:stream8 # We allow the CONFIG_TOML_SUFFIX to be overridden. From 256e37eb3f2a0fd415e97fab5f790adc84625af5 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 10 Aug 2022 11:57:36 +0200 Subject: [PATCH 2/8] Add fedora35 package targets Signed-off-by: Evan Lezar --- docker/docker.mk | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/docker/docker.mk b/docker/docker.mk index c21aa51b..64831113 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -107,6 +107,16 @@ LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG) --centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum --centos8%: BASEIMAGE = quay.io/centos/centos:stream8 +# private fedora target +--fedora%: OS := fedora +--fedora%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1) +--fedora%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1) +--fedora%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum +--fedora%: CONFIG_TOML_SUFFIX := centos +# The fedora(35) base image has very slow performance when building aarch64 packages. +# Since our primary concern here is glibc versions, we use the older glibc version available in centos8. +--fedora35%: BASEIMAGE = quay.io/centos/centos:stream8 + # private amazonlinux target --amazonlinux%: OS := amazonlinux --amazonlinux%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1) @@ -128,6 +138,7 @@ LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG) --rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum --rhel8%: BASEIMAGE = quay.io/centos/centos:stream8 + # We allow the CONFIG_TOML_SUFFIX to be overridden. CONFIG_TOML_SUFFIX ?= $(OS) From f17121fd6c09a7ba59a3eba777c8e04e4a12a536 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 10 Aug 2022 11:59:19 +0200 Subject: [PATCH 3/8] Add fedora targets to release scripts Signed-off-by: Evan Lezar --- scripts/build-packages.sh | 2 ++ scripts/release-packages.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/scripts/build-packages.sh b/scripts/build-packages.sh index 9d64fece..254d85bc 100755 --- a/scripts/build-packages.sh +++ b/scripts/build-packages.sh @@ -37,6 +37,8 @@ all=( centos8-x86_64 debian10-amd64 debian9-amd64 + fedora35-aarch64 + fedora35-x86_64 opensuse-leap15.1-x86_64 ubuntu16.04-amd64 ubuntu16.04-ppc64le diff --git a/scripts/release-packages.sh b/scripts/release-packages.sh index 25ca7bed..6b6a5bed 100755 --- a/scripts/release-packages.sh +++ b/scripts/release-packages.sh @@ -148,6 +148,8 @@ all=( centos8-x86_64 debian10-amd64 debian9-amd64 + fedora35-aarch64 + fedora35-x86_64 opensuse-leap15.1-x86_64 ubuntu16.04-amd64 ubuntu16.04-ppc64le From 22427c135927449ffc1c92d3e6f4ae9c382aa5f0 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 10 Aug 2022 12:02:40 +0200 Subject: [PATCH 4/8] Add fedora35 CI targets Signed-off-by: Evan Lezar --- .common-ci.yml | 4 ++++ .gitlab-ci.yml | 14 ++++++++++++++ 2 files changed, 18 insertions(+) diff --git a/.common-ci.yml b/.common-ci.yml index 44675bd6..e9bd5a5c 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -57,6 +57,10 @@ stages: variables: DIST: debian9 +.dist-fedora35: + variables: + DIST: fedora35 + .dist-opensuse-leap15.1: variables: DIST: opensuse-leap15.1 diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 919d6588..71261fc6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -158,6 +158,18 @@ package-debian9-amd64: - .dist-debian9 - .arch-amd64 +package-fedora35-aarch64: + extends: + - .package-build + - .dist-fedora35 + - .arch-aarch64 + +package-fedora35-x86_64: + extends: + - .package-build + - .dist-fedora35 + - .arch-x86_64 + package-opensuse-leap15.1-x86_64: extends: - .package-build @@ -278,6 +290,8 @@ image-packaging: - package-centos8-x86_64 - package-debian10-amd64 - package-debian9-amd64 + - package-fedora35-aarch64 + - package-fedora35-x86_64 - package-opensuse-leap15.1-x86_64 - package-ubuntu16.04-amd64 - package-ubuntu16.04-ppc64le From a5bbf613e820c21f6159ae5323c17bdafafde9f2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 10 Aug 2022 14:32:44 +0200 Subject: [PATCH 5/8] Use single config file for centos, al2, and fedora Signed-off-by: Evan Lezar --- config/config.toml.centos | 32 ------------------- ...g.toml.amazonlinux => config.toml.rpm-yum} | 0 docker/docker.mk | 5 ++- 3 files changed, 4 insertions(+), 33 deletions(-) delete mode 100644 config/config.toml.centos rename config/{config.toml.amazonlinux => config.toml.rpm-yum} (100%) diff --git a/config/config.toml.centos b/config/config.toml.centos deleted file mode 100644 index 225ec450..00000000 --- a/config/config.toml.centos +++ /dev/null @@ -1,32 +0,0 @@ -disable-require = false -#swarm-resource = "DOCKER_RESOURCE_GPU" -#accept-nvidia-visible-devices-envvar-when-unprivileged = true -#accept-nvidia-visible-devices-as-volume-mounts = false - -[nvidia-container-cli] -#root = "/run/nvidia/driver" -#path = "/usr/bin/nvidia-container-cli" -environment = [] -#debug = "/var/log/nvidia-container-toolkit.log" -#ldcache = "/etc/ld.so.cache" -load-kmods = true -#no-cgroups = false -#user = "root:video" -ldconfig = "@/sbin/ldconfig" - -[nvidia-container-runtime] -#debug = "/var/log/nvidia-container-runtime.log" -log-level = "info" - -# Specify the runtimes to consider. This list is processed in order and the PATH -# searched for matching executables unless the entry is an absolute path. -runtimes = [ - "docker-runc", - "runc", -] - -mode = "auto" - - [nvidia-container-runtime.modes.csv] - - mount-spec-path = "/etc/nvidia-container-runtime/host-files-for-container.d" diff --git a/config/config.toml.amazonlinux b/config/config.toml.rpm-yum similarity index 100% rename from config/config.toml.amazonlinux rename to config/config.toml.rpm-yum diff --git a/docker/docker.mk b/docker/docker.mk index 64831113..7ba6efd5 100644 --- a/docker/docker.mk +++ b/docker/docker.mk @@ -105,6 +105,7 @@ LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG) --centos%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1) --centos%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1) --centos%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum +--centos%: CONFIG_TOML_SUFFIX := rpm-yum --centos8%: BASEIMAGE = quay.io/centos/centos:stream8 # private fedora target @@ -112,7 +113,7 @@ LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG) --fedora%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1) --fedora%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1) --fedora%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum ---fedora%: CONFIG_TOML_SUFFIX := centos +--fedora%: CONFIG_TOML_SUFFIX := rpm-yum # The fedora(35) base image has very slow performance when building aarch64 packages. # Since our primary concern here is glibc versions, we use the older glibc version available in centos8. --fedora35%: BASEIMAGE = quay.io/centos/centos:stream8 @@ -122,6 +123,7 @@ LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG) --amazonlinux%: LIBNVIDIA_CONTAINER_TOOLS_VERSION := $(LIBNVIDIA_CONTAINER_VERSION)-$(if $(LIBNVIDIA_CONTAINER_TAG),0.1.$(LIBNVIDIA_CONTAINER_TAG),1) --amazonlinux%: PKG_REV := $(if $(LIB_TAG),0.1.$(LIB_TAG),1) --amazonlinux%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum +--amazonlinux%: CONFIG_TOML_SUFFIX := rpm-yum # private opensuse-leap target --opensuse-leap%: OS = opensuse-leap @@ -136,6 +138,7 @@ LIBNVIDIA_CONTAINER_TAG ?= $(LIB_TAG) --rhel%: VERSION = $(patsubst rhel%-$(ARCH),%,$(TARGET_PLATFORM)) --rhel%: ARTIFACTS_DIR = $(DIST_DIR)/rhel$(VERSION)/$(ARCH) --rhel%: DOCKERFILE = $(CURDIR)/docker/Dockerfile.rpm-yum +--rhel%: CONFIG_TOML_SUFFIX := rpm-yum --rhel8%: BASEIMAGE = quay.io/centos/centos:stream8 From 6a155cc606f4df43f921797b9ed9e11e184af84c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 11 Aug 2022 11:21:18 +0200 Subject: [PATCH 6/8] Increase package build timeout to 3 hours for slow aarch64 builds Signed-off-by: Evan Lezar --- .gitlab-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 71261fc6..177d5006 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -94,7 +94,7 @@ unit-tests: - .multi-arch-build - .package-artifacts stage: package-build - timeout: 2h 30m + timeout: 3h script: - ./scripts/build-packages.sh ${DIST}-${ARCH} From 3269a7b0e77e1daeb09afcea222c1f305ff297ff Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 23 Aug 2022 13:12:52 +0200 Subject: [PATCH 7/8] Update libnvidia-container submodule Signed-off-by: Evan Lezar --- third_party/libnvidia-container | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/libnvidia-container b/third_party/libnvidia-container index 72a9d90a..5c7ea1d6 160000 --- a/third_party/libnvidia-container +++ b/third_party/libnvidia-container @@ -1 +1 @@ -Subproject commit 72a9d90adca6dfb78b7ea8b270f8f5c64c3169e4 +Subproject commit 5c7ea1d6bfa727e20e4feded4a3d689ab5beb56c From ad7151d3943846bc01c005f0ad6f8e9eafb23fb2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 23 Aug 2022 14:16:14 +0200 Subject: [PATCH 8/8] Update CUDA base image to 11.7.1 Signed-off-by: Evan Lezar --- versions.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.mk b/versions.mk index e3dfe63b..b993813b 100644 --- a/versions.mk +++ b/versions.mk @@ -24,7 +24,7 @@ NVIDIA_CONTAINER_RUNTIME_VERSION := 3.11.0 # Specify the expected libnvidia-container0 version for arm64-based ubuntu builds. LIBNVIDIA_CONTAINER0_VERSION := 0.10.0+jetpack -CUDA_VERSION := 11.7.0 +CUDA_VERSION := 11.7.1 GOLANG_VERSION := 1.17.8 GIT_COMMIT ?= $(shell git describe --dirty --long --always 2> /dev/null || echo "")