From abb0b7be5de0f20403885af209161b7e30e1c890 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 20 Jan 2022 17:42:19 +0100 Subject: [PATCH] Add scripting to sign and publish packages Signed-off-by: Evan Lezar --- DEVELOPMENT.md | 24 +++- scripts/Dockerfile.sign.deb | 4 + scripts/Dockerfile.sign.rpm | 3 + scripts/packages-sign-all.sh | 95 +++++++++++++++ scripts/release-packages.sh | 219 +++++++++++++++++++++++++++++++++++ 5 files changed, 340 insertions(+), 5 deletions(-) create mode 100644 scripts/Dockerfile.sign.deb create mode 100644 scripts/Dockerfile.sign.rpm create mode 100755 scripts/packages-sign-all.sh create mode 100755 scripts/release-packages.sh diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index ff070c62..3d9419d1 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -39,9 +39,23 @@ The [test/release](./test/release/) folder contains documentation on how the ins ## Releasing -A utility script [`scripts/release.sh`](./scripts/release.sh) is provided to build -packages required for release. If run without arguments, all supported distribution-architecture combinations are built. A specific distribution-architecture pair can also be provided -```sh -./scripts/release.sh ubuntu18.04-amd64 +In order to release packages required for a release, a utility script +[`scripts/release-packages.sh`](./scripts/release-packages.sh) is provided. +This script can be executed as follows: + +```bash +GPG_LOCAL_USER="GPG_USER" \ +MASTER_KEY_PATH=/path/to/gpg-master.key \ +SUB_KEY_PATH=/path/to/gpg-subkey.key \ + ./scripts/release-packages.sh REPO PACKAGE_REPO_ROOT [REFERENCE] ``` -where the `amd64` builds for `ubuntu18.04` are provided as an example. + +Where `REPO` is one of `stable` or `experimental`, `PACKAGE_REPO_ROOT` is the local path to the `libnvidia-container` repository checked out to the `gh-pages` branch, and `REFERENCE` is the git SHA that is to be released. If reference is not specified `HEAD` is assumed. + +This scripts performs the following basic functions: +* Pulls the package image defined by the `REFERENCE` git SHA from the staging registry, +* Copies the required packages to the package repository at `PACKAGE_REPO_ROOT/REPO`, +* Signs the packages using the specified GPG keys + +While the last two are performed, commits are added to the package repository. These can be pushed to the relevant repository. + diff --git a/scripts/Dockerfile.sign.deb b/scripts/Dockerfile.sign.deb new file mode 100644 index 00000000..ba117354 --- /dev/null +++ b/scripts/Dockerfile.sign.deb @@ -0,0 +1,4 @@ +FROM ubuntu:18.04 + +RUN apt-get update && \ + apt-get install -y apt-utils gpg xz-utils diff --git a/scripts/Dockerfile.sign.rpm b/scripts/Dockerfile.sign.rpm new file mode 100644 index 00000000..900cd305 --- /dev/null +++ b/scripts/Dockerfile.sign.rpm @@ -0,0 +1,3 @@ +FROM centos:8 + +RUN yum install -y createrepo rpm-sign pinentry diff --git a/scripts/packages-sign-all.sh b/scripts/packages-sign-all.sh new file mode 100755 index 00000000..1b660feb --- /dev/null +++ b/scripts/packages-sign-all.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash + +: "${ALL_DEBS:? Must set ALL_DEBS}" +: "${ALL_RPMS:? Must set ALL_RPMS}" +: "${GPG_LOCAL_USER:? Must set GPG_LOCAL_USER}" +: "${TARGETS:? Must set TARGETS}" + +set -x -e + +function deb-sign { + local last_found + for r in ${*}; do + if [ -f "./${r}" ]; then + last_found=${r} + fi + done + if [[ -z ${last_found} ]]; then + echo "WARNING: No expected package found in $(pwd); skipping signing of repo;" + return + fi + apt-ftparchive packages . \ + | tee Packages \ + | xz > Packages.xz + apt-ftparchive -c repo.conf release . \ + | gpg --batch --yes --expert --clearsign \ + --armor \ + --no-emit-version \ + --no-comments \ + --personal-digest-preferences sha512 \ + --local-user ${GPG_LOCAL_USER} \ + > InRelease +} + +function rpm-sign { + for r in ${*}; do + if [ -f "./${r}" ]; then + rpmsign --addsign --key-id A04EA552 --digest-algo=sha512 "${r}" + fi + done + createrepo -v --no-database -s sha512 --compress-type xz --revision "1.0" . + gpg2 --batch --yes --expert --sign --detach-sign \ + --armor \ + --no-emit-version \ + --no-comments --personal-digest-preferences sha512 \ + --local-user ${GPG_LOCAL_USER} \ + repodata/repomd.xml +} + +function sign() { + local target=$1 + local dst_root=$2 + + local src_dist=${target%-*} + local dist=${src_dist/amazonlinux/amzn} + + local pkg_type + case ${src_dist} in + amazonlinux*) pkg_type=rpm + ;; + centos*) pkg_type=rpm + ;; + debian*) pkg_type=deb + ;; + opensuse-leap*) pkg_type=rpm + ;; + ubuntu*) pkg_type=deb + ;; + *) echo "ERROR: unexpected distribution ${src_dist}" + ;; + esac + + local arch=${target##*-} + case ${src_dist} in + ubuntu*) arch=${arch//ppc64le/ppc64el} + esac + + local dst=${dst_root}/${dist}/${arch} + + if [[ ! -d ${dst} ]]; then + echo "Directory ${dst} not found. Skipping" + return + fi + + cd ${dst} + if [[ -f "/etc/debian_version" ]]; then + [[ ${pkg_type} == "deb" ]] && deb-sign ${ALL_DEBS} + else + [[ ${pkg_type} == "rpm" ]] && rpm-sign ${ALL_RPMS} + fi + cd - +} + +for target in ${TARGETS[@]}; do + sign ${target} $(pwd) +done diff --git a/scripts/release-packages.sh b/scripts/release-packages.sh new file mode 100755 index 00000000..4007c810 --- /dev/null +++ b/scripts/release-packages.sh @@ -0,0 +1,219 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function assert_usage() { + echo "Incorrect arguments: $*" + echo "$(basename ${BASH_SOURCE[0]}) REPO PACKAGE_REPO_ROOT [SHA]" + echo "\tSHA: The SHA / reference to release. [Default: HEAD]" + exit 1 +} + +set -e + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" +PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )" + +if [[ $# -lt 1 || $# -gt 2 ]]; then + assert_usage $* +fi + +PACKAGE_REPO_ROOT=$1 +if [[ ! -d ${PACKAGE_REPO_ROOT} ]]; then + echo "The specified PACKAGE_REPO_ROOT '${PACKAGE_REPO_ROOT}' must exist" + exit 1 +fi + +: ${REFERENCE:="HEAD"} +if [[ $# -ge 2 ]]; then + REFERENCE=$2 +fi + +eval $(${SCRIPTS_DIR}/get-component-versions.sh) + +TAG=v"${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" +SHA=$(git rev-parse --short=8 ${REFERENCE}) + +REPO="experimental" +if [[ ${TAG/rc./} == ${TAG} ]]; then + REPO="stable" +fi + +PACKAGE_CACHE=release-${TAG}-${REPO} + +echo "Fetching packages with SHA '${SHA}' as tag '${TAG}' to ${PACKAGE_CACHE}" +IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit" +IMAGE_TAG=${SHA}-packaging +${SCRIPTS_DIR}/pull-packages.sh \ + ${IMAGE_NAME}:${IMAGE_TAG} \ + ${PACKAGE_CACHE} + +: ${ALL_RPMS:="$(find ${PACKAGE_CACHE} -name "*.rpm" -exec basename {} \; | sort | uniq)"} +: ${ALL_DEBS:="$(find ${PACKAGE_CACHE} -name "*.deb" -exec basename {} \; | sort | uniq)"} + + +PACKAGE_REPO_ROOT=$(cd "${PACKAGE_REPO_ROOT}" && pwd) +echo "Updating ${REPO} repo at ${PACKAGE_REPO_ROOT}" + +docker build \ + -t nvidia/toolkit-deb-pkg-signer \ + -f ${SCRIPTS_DIR}/Dockerfile.sign.deb \ + ${SCRIPTS_DIR} + +docker build \ + -t nvidia/toolkit-rpm-pkg-signer \ + -f ${SCRIPTS_DIR}/Dockerfile.sign.rpm \ + ${SCRIPTS_DIR} + +function sync() { + local target=$1 + local src_root=$2 + local dst_root=$3 + + local src_dist=${target%-*} + local dst_dist=${src_dist/amazonlinux/amzn} + + local pkg_type + case ${src_dist} in + amazonlinux*) pkg_type=rpm + ;; + centos*) pkg_type=rpm + ;; + debian*) pkg_type=deb + ;; + opensuse-leap*) pkg_type=rpm + ;; + ubuntu*) pkg_type=deb + ;; + *) echo "ERROR: unexpected distribution ${src_dist}" + ;; + esac + + local arch=${target##*-} + local dst_arch=${arch} + case ${src_dist} in + ubuntu*) dst_arch=${arch//ppc64le/ppc64el} + esac + + local src=${src_root}/${src_dist}/${arch} + local dst=${dst_root}/${dst_dist}/${dst_arch} + + if [[ ! -d ${src} || -z $(ls ${src}/*.${pkg_type}) ]]; then + echo "Skipping ${src}" + return + fi + mkdir -p ${dst} + cp ${src}/libnvidia-container*.${pkg_type} ${dst} + cp ${src}/nvidia-container-toolkit*.${pkg_type} ${dst} + if [[ ${REPO} == "stable" ]]; then + cp ${src}/nvidia-container-runtime*.${pkg_type} ${dst} + cp ${src}/nvidia-docker*.${pkg_type} ${dst} + fi +} + +# This list represents the distribution-architecture pairs that are actually published +# to the relevant repositories. This targets forwarded to the build-all-components script +# can be overridden by specifying command line arguments. +all=( + amazonlinux2-aarch64 + amazonlinux2-x86_64 + centos7-ppc64le + centos7-x86_64 + centos8-aarch64 + centos8-ppc64le + centos8-x86_64 + debian10-amd64 + debian9-amd64 + opensuse-leap15.1-x86_64 + ubuntu16.04-amd64 + ubuntu16.04-ppc64le + ubuntu18.04-amd64 + ubuntu18.04-arm64 + ubuntu18.04-ppc64le +) + +targets=${all[@]} + +: ${UPSTREAM_REFERENCE:="upstream/gh-pages"} +git -C ${PACKAGE_REPO_ROOT} reset --hard ${UPSTREAM_REFERENCE} +git -C ${PACKAGE_REPO_ROOT} clean -fdx + +for target in ${targets[@]}; do + sync ${target} ${PACKAGE_CACHE} ${PACKAGE_REPO_ROOT}/${REPO} +done + +git -C ${PACKAGE_REPO_ROOT} add ${REPO} + +if [[ ${REPO} == "stable" ]]; then +# Stable release +git -C ${PACKAGE_REPO_ROOT} commit -s -F- <