From 6709da4cea2d26eab0c0dedc2959b66d01ec033c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 28 Jan 2022 17:10:46 +0100 Subject: [PATCH 1/4] Rename release.sh to build-packages.sh The name release.sh is overloaded. This change renames the script to make the intent clearer. Signed-off-by: Evan Lezar --- .gitlab-ci.yml | 2 +- DEVELOPMENT.md | 4 +++- scripts/{release.sh => build-packages.sh} | 0 test/release/Makefile | 2 +- 4 files changed, 5 insertions(+), 3 deletions(-) rename scripts/{release.sh => build-packages.sh} (100%) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86e893d2..9716b97d 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -96,7 +96,7 @@ unit-tests: stage: package-build timeout: 2h 30m script: - - ./scripts/release.sh ${DIST}-${ARCH} + - ./scripts/build-packages.sh ${DIST}-${ARCH} artifacts: name: ${ARTIFACTS_NAME} diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 33d48894..ff070c62 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -13,7 +13,7 @@ The `nvidia-container-toolkit` resides in this repo directly. In oder to build the packages, the following command is executed ```sh -./scripts/build-all-components.sh TARGET +./scripts/build-packages.sh TARGET ``` where `TARGET` is a make target that is valid for each of the sub-components. @@ -21,6 +21,8 @@ These include: * `ubuntu18.04-amd64` * `centos8-x86_64` +If no `TARGET` is specified, all valid release targets are built. + The packages are generated in the `dist` folder. ## Testing local changes diff --git a/scripts/release.sh b/scripts/build-packages.sh similarity index 100% rename from scripts/release.sh rename to scripts/build-packages.sh diff --git a/test/release/Makefile b/test/release/Makefile index 3436bf69..37cbee25 100644 --- a/test/release/Makefile +++ b/test/release/Makefile @@ -56,4 +56,4 @@ $(RUN_TARGETS): run-%: image-% # Ensure that the local package root exists $(RELEASE_TARGETS): release-%: $(LOCAL_PACKAGE_ROOT)/$(*)/$(ARCH) - $(PROJECT_ROOT)/scripts/release.sh $(*)-$(ARCH) \ No newline at end of file + $(PROJECT_ROOT)/scripts/build-packages.sh $(*)-$(ARCH) From c09e5aca7774fce4a0b7e108edffd086f376e5d1 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 28 Jan 2022 13:51:39 +0100 Subject: [PATCH 2/4] Add envvar for package versions Signed-off-by: Evan Lezar --- scripts/get-component-versions.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/scripts/get-component-versions.sh b/scripts/get-component-versions.sh index b14e092e..80e34a51 100755 --- a/scripts/get-component-versions.sh +++ b/scripts/get-component-versions.sh @@ -54,12 +54,16 @@ nvidia_docker_tag=${nvidia_container_toolkit_tag} nvidia_docker_version_tag="${nvidia_docker_version}${nvidia_docker_tag:+~${nvidia_docker_tag}}" echo "LIBNVIDIA_CONTAINER_VERSION=${libnvidia_container_version_tag}" +echo "LIBNVIDIA_CONTAINER_PACKAGE_VERSION=${libnvidia_container_version_tag//\~/-}" echo "NVIDIA_CONTAINER_TOOLKIT_VERSION=${nvidia_container_toolkit_version}" echo "NVIDIA_CONTAINER_TOOLKIT_TAG=${nvidia_container_toolkit_tag}" +echo "NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION=${nvidia_container_toolkit_version_tag//\~/-}" if [[ "${libnvidia_container_version_tag}" != "${nvidia_container_toolkit_version_tag}" ]]; then >&2 echo "WARNING: The libnvidia-container and nvidia-container-toolkit versions do not match" fi echo "NVIDIA_CONTAINER_RUNTIME_VERSION=${nvidia_container_runtime_version}" echo "NVIDIA_CONTAINER_RUNTIME_TAG=${nvidia_container_runtime_tag}" +echo "NVIDIA_CONTAINER_RUNTIME_PACKAGE_VERSION=${nvidia_container_runtime_version_tag//\~/-}" echo "NVIDIA_DOCKER_VERSION=${nvidia_docker_version}" echo "NVIDIA_DOCKER_TAG=${nvidia_docker_tag}" +echo "NVIDIA_DOCKER_PACKAGE_VERSION=${nvidia_docker_version_tag//\~/-}" From abb0b7be5de0f20403885af209161b7e30e1c890 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 20 Jan 2022 17:42:19 +0100 Subject: [PATCH 3/4] Add scripting to sign and publish packages Signed-off-by: Evan Lezar --- DEVELOPMENT.md | 24 +++- scripts/Dockerfile.sign.deb | 4 + scripts/Dockerfile.sign.rpm | 3 + scripts/packages-sign-all.sh | 95 +++++++++++++++ scripts/release-packages.sh | 219 +++++++++++++++++++++++++++++++++++ 5 files changed, 340 insertions(+), 5 deletions(-) create mode 100644 scripts/Dockerfile.sign.deb create mode 100644 scripts/Dockerfile.sign.rpm create mode 100755 scripts/packages-sign-all.sh create mode 100755 scripts/release-packages.sh diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index ff070c62..3d9419d1 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -39,9 +39,23 @@ The [test/release](./test/release/) folder contains documentation on how the ins ## Releasing -A utility script [`scripts/release.sh`](./scripts/release.sh) is provided to build -packages required for release. If run without arguments, all supported distribution-architecture combinations are built. A specific distribution-architecture pair can also be provided -```sh -./scripts/release.sh ubuntu18.04-amd64 +In order to release packages required for a release, a utility script +[`scripts/release-packages.sh`](./scripts/release-packages.sh) is provided. +This script can be executed as follows: + +```bash +GPG_LOCAL_USER="GPG_USER" \ +MASTER_KEY_PATH=/path/to/gpg-master.key \ +SUB_KEY_PATH=/path/to/gpg-subkey.key \ + ./scripts/release-packages.sh REPO PACKAGE_REPO_ROOT [REFERENCE] ``` -where the `amd64` builds for `ubuntu18.04` are provided as an example. + +Where `REPO` is one of `stable` or `experimental`, `PACKAGE_REPO_ROOT` is the local path to the `libnvidia-container` repository checked out to the `gh-pages` branch, and `REFERENCE` is the git SHA that is to be released. If reference is not specified `HEAD` is assumed. + +This scripts performs the following basic functions: +* Pulls the package image defined by the `REFERENCE` git SHA from the staging registry, +* Copies the required packages to the package repository at `PACKAGE_REPO_ROOT/REPO`, +* Signs the packages using the specified GPG keys + +While the last two are performed, commits are added to the package repository. These can be pushed to the relevant repository. + diff --git a/scripts/Dockerfile.sign.deb b/scripts/Dockerfile.sign.deb new file mode 100644 index 00000000..ba117354 --- /dev/null +++ b/scripts/Dockerfile.sign.deb @@ -0,0 +1,4 @@ +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get install -y apt-utils gpg xz-utils diff --git a/scripts/Dockerfile.sign.rpm b/scripts/Dockerfile.sign.rpm new file mode 100644 index 00000000..900cd305 --- /dev/null +++ b/scripts/Dockerfile.sign.rpm @@ -0,0 +1,3 @@ +FROM centos:8 + +RUN yum install -y createrepo rpm-sign pinentry diff --git a/scripts/packages-sign-all.sh b/scripts/packages-sign-all.sh new file mode 100755 index 00000000..1b660feb --- /dev/null +++ b/scripts/packages-sign-all.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +: "${ALL_DEBS:? Must set ALL_DEBS}" +: "${ALL_RPMS:? Must set ALL_RPMS}" +: "${GPG_LOCAL_USER:? Must set GPG_LOCAL_USER}" +: "${TARGETS:? Must set TARGETS}" + +set -x -e + +function deb-sign { + local last_found + for r in ${*}; do + if [ -f "./${r}" ]; then + last_found=${r} + fi + done + if [[ -z ${last_found} ]]; then + echo "WARNING: No expected package found in $(pwd); skipping signing of repo;" + return + fi + apt-ftparchive packages . \ + | tee Packages \ + | xz > Packages.xz + apt-ftparchive -c repo.conf release . \ + | gpg --batch --yes --expert --clearsign \ + --armor \ + --no-emit-version \ + --no-comments \ + --personal-digest-preferences sha512 \ + --local-user ${GPG_LOCAL_USER} \ + > InRelease +} + +function rpm-sign { + for r in ${*}; do + if [ -f "./${r}" ]; then + rpmsign --addsign --key-id A04EA552 --digest-algo=sha512 "${r}" + fi + done + createrepo -v --no-database -s sha512 --compress-type xz --revision "1.0" . + gpg2 --batch --yes --expert --sign --detach-sign \ + --armor \ + --no-emit-version \ + --no-comments --personal-digest-preferences sha512 \ + --local-user ${GPG_LOCAL_USER} \ + repodata/repomd.xml +} + +function sign() { + local target=$1 + local dst_root=$2 + + local src_dist=${target%-*} + local dist=${src_dist/amazonlinux/amzn} + + local pkg_type + case ${src_dist} in + amazonlinux*) pkg_type=rpm + ;; + centos*) pkg_type=rpm + ;; + debian*) pkg_type=deb + ;; + opensuse-leap*) pkg_type=rpm + ;; + ubuntu*) pkg_type=deb + ;; + *) echo "ERROR: unexpected distribution ${src_dist}" + ;; + esac + + local arch=${target##*-} + case ${src_dist} in + ubuntu*) arch=${arch//ppc64le/ppc64el} + esac + + local dst=${dst_root}/${dist}/${arch} + + if [[ ! -d ${dst} ]]; then + echo "Directory ${dst} not found. Skipping" + return + fi + + cd ${dst} + if [[ -f "/etc/debian_version" ]]; then + [[ ${pkg_type} == "deb" ]] && deb-sign ${ALL_DEBS} + else + [[ ${pkg_type} == "rpm" ]] && rpm-sign ${ALL_RPMS} + fi + cd - +} + +for target in ${TARGETS[@]}; do + sign ${target} $(pwd) +done diff --git a/scripts/release-packages.sh b/scripts/release-packages.sh new file mode 100755 index 00000000..4007c810 --- /dev/null +++ b/scripts/release-packages.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function assert_usage() { + echo "Incorrect arguments: $*" + echo "$(basename ${BASH_SOURCE[0]}) REPO PACKAGE_REPO_ROOT [SHA]" + echo "\tSHA: The SHA / reference to release. [Default: HEAD]" + exit 1 +} + +set -e + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" +PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )" + +if [[ $# -lt 1 || $# -gt 2 ]]; then + assert_usage $* +fi + +PACKAGE_REPO_ROOT=$1 +if [[ ! -d ${PACKAGE_REPO_ROOT} ]]; then + echo "The specified PACKAGE_REPO_ROOT '${PACKAGE_REPO_ROOT}' must exist" + exit 1 +fi + +: ${REFERENCE:="HEAD"} +if [[ $# -ge 2 ]]; then + REFERENCE=$2 +fi + +eval $(${SCRIPTS_DIR}/get-component-versions.sh) + +TAG=v"${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" +SHA=$(git rev-parse --short=8 ${REFERENCE}) + +REPO="experimental" +if [[ ${TAG/rc./} == ${TAG} ]]; then + REPO="stable" +fi + +PACKAGE_CACHE=release-${TAG}-${REPO} + +echo "Fetching packages with SHA '${SHA}' as tag '${TAG}' to ${PACKAGE_CACHE}" +IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit" +IMAGE_TAG=${SHA}-packaging +${SCRIPTS_DIR}/pull-packages.sh \ + ${IMAGE_NAME}:${IMAGE_TAG} \ + ${PACKAGE_CACHE} + +: ${ALL_RPMS:="$(find ${PACKAGE_CACHE} -name "*.rpm" -exec basename {} \; | sort | uniq)"} +: ${ALL_DEBS:="$(find ${PACKAGE_CACHE} -name "*.deb" -exec basename {} \; | sort | uniq)"} + + +PACKAGE_REPO_ROOT=$(cd "${PACKAGE_REPO_ROOT}" && pwd) +echo "Updating ${REPO} repo at ${PACKAGE_REPO_ROOT}" + +docker build \ + -t nvidia/toolkit-deb-pkg-signer \ + -f ${SCRIPTS_DIR}/Dockerfile.sign.deb \ + ${SCRIPTS_DIR} + +docker build \ + -t nvidia/toolkit-rpm-pkg-signer \ + -f ${SCRIPTS_DIR}/Dockerfile.sign.rpm \ + ${SCRIPTS_DIR} + +function sync() { + local target=$1 + local src_root=$2 + local dst_root=$3 + + local src_dist=${target%-*} + local dst_dist=${src_dist/amazonlinux/amzn} + + local pkg_type + case ${src_dist} in + amazonlinux*) pkg_type=rpm + ;; + centos*) pkg_type=rpm + ;; + debian*) pkg_type=deb + ;; + opensuse-leap*) pkg_type=rpm + ;; + ubuntu*) pkg_type=deb + ;; + *) echo "ERROR: unexpected distribution ${src_dist}" + ;; + esac + + local arch=${target##*-} + local dst_arch=${arch} + case ${src_dist} in + ubuntu*) dst_arch=${arch//ppc64le/ppc64el} + esac + + local src=${src_root}/${src_dist}/${arch} + local dst=${dst_root}/${dst_dist}/${dst_arch} + + if [[ ! -d ${src} || -z $(ls ${src}/*.${pkg_type}) ]]; then + echo "Skipping ${src}" + return + fi + mkdir -p ${dst} + cp ${src}/libnvidia-container*.${pkg_type} ${dst} + cp ${src}/nvidia-container-toolkit*.${pkg_type} ${dst} + if [[ ${REPO} == "stable" ]]; then + cp ${src}/nvidia-container-runtime*.${pkg_type} ${dst} + cp ${src}/nvidia-docker*.${pkg_type} ${dst} + fi +} + +# This list represents the distribution-architecture pairs that are actually published +# to the relevant repositories. This targets forwarded to the build-all-components script +# can be overridden by specifying command line arguments. +all=( + amazonlinux2-aarch64 + amazonlinux2-x86_64 + centos7-ppc64le + centos7-x86_64 + centos8-aarch64 + centos8-ppc64le + centos8-x86_64 + debian10-amd64 + debian9-amd64 + opensuse-leap15.1-x86_64 + ubuntu16.04-amd64 + ubuntu16.04-ppc64le + ubuntu18.04-amd64 + ubuntu18.04-arm64 + ubuntu18.04-ppc64le +) + +targets=${all[@]} + +: ${UPSTREAM_REFERENCE:="upstream/gh-pages"} +git -C ${PACKAGE_REPO_ROOT} reset --hard ${UPSTREAM_REFERENCE} +git -C ${PACKAGE_REPO_ROOT} clean -fdx + +for target in ${targets[@]}; do + sync ${target} ${PACKAGE_CACHE} ${PACKAGE_REPO_ROOT}/${REPO} +done + +git -C ${PACKAGE_REPO_ROOT} add ${REPO} + +if [[ ${REPO} == "stable" ]]; then +# Stable release +git -C ${PACKAGE_REPO_ROOT} commit -s -F- < Date: Tue, 22 Mar 2022 16:37:26 +0200 Subject: [PATCH 4/4] Improve handling of git remotes for gh-pages packages Signed-off-by: Evan Lezar --- scripts/release-packages.sh | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/scripts/release-packages.sh b/scripts/release-packages.sh index 4007c810..88bce898 100755 --- a/scripts/release-packages.sh +++ b/scripts/release-packages.sh @@ -16,7 +16,8 @@ function assert_usage() { echo "Incorrect arguments: $*" - echo "$(basename ${BASH_SOURCE[0]}) REPO PACKAGE_REPO_ROOT [SHA]" + echo "$(basename ${BASH_SOURCE[0]}) PACKAGE_REPO_ROOT [SHA]" + echo "\tPACKAGE_REPO_ROOT: The path to the libnvidia-container repository" echo "\tSHA: The SHA / reference to release. [Default: HEAD]" exit 1 } @@ -146,9 +147,22 @@ all=( targets=${all[@]} -: ${UPSTREAM_REFERENCE:="upstream/gh-pages"} +_current_branch=$(git -C ${PACKAGE_REPO_ROOT} rev-parse --abbrev-ref HEAD) +if [[ x"${_current_branch}" != x"gh-pages" ]]; then + echo "It is expected that the gh-pages branch be checked out" + exit 1 +fi + +: ${UPSTREAM_REMOTE:="origin"} +_remote_name=$( git remote -v | grep "git@gitlab.com:nvidia/container-toolkit/libnvidia-container.git (push)" | cut -d$'\t' -f1 ) +if [[ x"${_remote_name}" != x"${UPSTREAM_REMOTE}" ]]; then + echo "Identified ${_remote_name} as git@gitlab.com:nvidia/container-toolkit/libnvidia-container.git remote." + echo "Set UPSTREAM_REMOTE=${_remote_name} instead of ${UPSTREAM_REMOTE}" +fi + +: ${UPSTREAM_REFERENCE:="${UPSTREAM_REMOTE}/gh-pages"} git -C ${PACKAGE_REPO_ROOT} reset --hard ${UPSTREAM_REFERENCE} -git -C ${PACKAGE_REPO_ROOT} clean -fdx +git -C ${PACKAGE_REPO_ROOT} clean -fdx ${REPO} for target in ${targets[@]}; do sync ${target} ${PACKAGE_CACHE} ${PACKAGE_REPO_ROOT}/${REPO}