From 39b0830a661a691f9412a1a8b94798d96676e3c2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 8 May 2023 15:36:24 +0200 Subject: [PATCH 1/3] Extract manifest from packaging image Also include manifest.txt with, for example, version info when extracting packages from the packagin image. Signed-off-by: Evan Lezar --- scripts/pull-packages.sh | 2 +- scripts/release-packages.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/pull-packages.sh b/scripts/pull-packages.sh index f62e0bb4..f13750ee 100755 --- a/scripts/pull-packages.sh +++ b/scripts/pull-packages.sh @@ -55,4 +55,4 @@ docker run --rm \ -u $(id -u):$(id -g) \ --entrypoint="bash" \ ${IMAGE} \ - -c "cp -R /artifacts/packages/* ${DIST_DIR}" + -c "cp --preserve=timestamps -R /artifacts/* ${DIST_DIR}" diff --git a/scripts/release-packages.sh b/scripts/release-packages.sh index c82e6a5f..ea0ebb23 100755 --- a/scripts/release-packages.sh +++ b/scripts/release-packages.sh @@ -189,7 +189,7 @@ git -C ${PACKAGE_REPO_ROOT} reset --hard ${UPSTREAM_REFERENCE} git -C ${PACKAGE_REPO_ROOT} clean -fdx ${REPO} for target in ${targets[@]}; do - sync ${target} ${PACKAGE_CACHE} ${PACKAGE_REPO_ROOT}/${REPO} + sync ${target} ${PACKAGE_CACHE}/packages ${PACKAGE_REPO_ROOT}/${REPO} done git -C ${PACKAGE_REPO_ROOT} add ${REPO} From 1c1ce2c6f78a51f8de1046f7fb35b687267500b8 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 8 May 2023 15:40:42 +0200 Subject: [PATCH 2/3] Use version from manifest to extract packages Signed-off-by: Evan Lezar --- scripts/release-kitmaker-artifactory.sh | 9 ---- scripts/release-packages.sh | 25 ++++++----- scripts/utils.sh | 59 +++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 21 deletions(-) diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh index d7bee6dc..1d5c9c76 100755 --- a/scripts/release-kitmaker-artifactory.sh +++ b/scripts/release-kitmaker-artifactory.sh @@ -67,13 +67,6 @@ fi KITMAKER_SCRATCH="${KITMAKER_DIR}/.scratch" -# extract_info extracts the value of the specified variable from the manifest.txt file. -function extract_info() { - local variable=$1 - local value=$(cat "${ARTIFACTS_DIR}/manifest.txt" | grep "#${variable}=" | sed -e "s/#${variable}=//" | tr -d '\r') - echo $value -} - IMAGE_EPOCH=$(extract_info "IMAGE_EPOCH") # Note we use the main branch for the kitmaker archive. GIT_BRANCH=main @@ -129,8 +122,6 @@ function create_archive() { rmdir "${scratch_dir}" } -function join_by { local IFS="$1"; shift; echo "$*"; } - function optionally_add_property() { local property=$1 local value=$2 diff --git a/scripts/release-packages.sh b/scripts/release-packages.sh index ea0ebb23..ccc73026 100755 --- a/scripts/release-packages.sh +++ b/scripts/release-packages.sh @@ -31,6 +31,8 @@ if [[ $# -lt 1 || $# -gt 2 ]]; then assert_usage $* fi +source "${SCRIPTS_DIR}"/utils.sh + PACKAGE_REPO_ROOT=$1 if [[ ! -d ${PACKAGE_REPO_ROOT} ]]; then echo "The specified PACKAGE_REPO_ROOT '${PACKAGE_REPO_ROOT}' must exist" @@ -42,21 +44,20 @@ if [[ $# -ge 2 ]]; then REFERENCE=$2 fi -eval $(${SCRIPTS_DIR}/get-component-versions.sh) - -TAG=v"${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" SHA=$(git rev-parse --short=8 ${REFERENCE}) +IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit" +IMAGE_TAG=${SHA}-packaging + +VERSION="$(get_version_from_image ${IMAGE_NAME}:${IMAGE_TAG} ${SHA})" REPO="experimental" -if [[ ${TAG/rc./} == ${TAG} ]]; then +if [[ ${VERSION/rc./} == ${VERSION} ]]; then REPO="stable" fi -PACKAGE_CACHE=release-${TAG}-${REPO} +PACKAGE_CACHE=release-${VERSION}-${REPO} -echo "Fetching packages with SHA '${SHA}' as tag '${TAG}' to ${PACKAGE_CACHE}" -IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit" -IMAGE_TAG=${SHA}-packaging +echo "Fetching packages with SHA '${SHA}' as tag '${VERSION}' to ${PACKAGE_CACHE}" ${SCRIPTS_DIR}/pull-packages.sh \ ${IMAGE_NAME}:${IMAGE_TAG} \ ${PACKAGE_CACHE} @@ -197,7 +198,7 @@ git -C ${PACKAGE_REPO_ROOT} add ${REPO} if [[ ${REPO} == "stable" ]]; then # Stable release git -C ${PACKAGE_REPO_ROOT} commit -s -F- < /dev/null; then + regctl image get-file "${image}" "${path_in_image}" "${path_on_host}" + else + # Note this will only work for destinations where the `path_on_host` is in `pwd` + docker run --rm \ + -v "$(pwd):$(pwd)" \ + -w "$(pwd)" \ + -u "$(id -u):$(id -g)" \ + --entrypoint="bash" \ + "${image}" \ + -c "cp ${path_in_image} ${path_on_host}" + fi +} + +# extract_info extracts the value of the specified variable from the manifest.txt file. +function extract_from_manifest() { + local variable=$1 + local manifest=$2 + local value=$(cat ${manifest} | grep "#${variable}=" | sed -e "s/#${variable}=//" | tr -d '\r') + echo $value +} + +# extract_info extracts the value of the specified variable from the manifest.txt file. +function extract_info() { + extract_from_manifest $1 "${ARTIFACTS_DIR}/manifest.txt" +} + +function get_version_from_image() { + local image=$1 + local manifest="manifest-${2}.txt" + copy_file ${image} "/artifacts/manifest.txt" ${manifest} + version=$(extract_from_manifest "PACKAGE_VERSION" ${manifest}) + echo "v${version/\~/-}" + rm -f ${manifest} +} + +function join_by { local IFS="$1"; shift; echo "$*"; } \ No newline at end of file From 3610b5073b39f3db8837d18aeaa62d187f1c56ea Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 8 May 2023 15:41:02 +0200 Subject: [PATCH 3/3] Add package release archive Signed-off-by: Evan Lezar --- .nvidia-ci.yml | 18 +++++ scripts/archive-packages.sh | 138 ++++++++++++++++++++++++++++++++++++ 2 files changed, 156 insertions(+) create mode 100755 scripts/archive-packages.sh diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index 366814b5..7260c114 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -37,6 +37,7 @@ variables: STAGING_VERSION: ${CI_COMMIT_SHORT_SHA} ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative" KITMAKER_RELEASE_FOLDER: "kitmaker" + PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases" .image-pull: stage: image-build @@ -216,6 +217,23 @@ release:packages:kitmaker: extends: - .release:packages +release:archive: + extends: + - .release:external + needs: + - image-packaging + variables: + VERSION: "${CI_COMMIT_SHORT_SHA}" + PACKAGE_REGISTRY: "${CI_REGISTRY}" + PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}" + PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}" + PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" + PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging" + PACKAGE_ARCHIVE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${PACKAGE_ARCHIVE_RELEASE_FOLDER}" + script: + - apk add --no-cache bash git + - ./scripts/archive-packages.sh "${PACKAGE_ARCHIVE_ARTIFACTORY_REPO}" + release:staging-ubuntu20.04: extends: - .release:staging diff --git a/scripts/archive-packages.sh b/scripts/archive-packages.sh new file mode 100755 index 00000000..5cbb5585 --- /dev/null +++ b/scripts/archive-packages.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +function assert_usage() { + echo "Incorrect arguments: $*" + echo "$(basename ${BASH_SOURCE[0]}) ARTIFACTORY_REPO GIT_REFERENCE" + exit 1 +} + +set -e + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" +PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )" + +if [[ $# -le 1 ]]; then + assert_usage "$@" +fi + +source "${SCRIPTS_DIR}"/utils.sh + +ARTIFACTORY_REPO=$1 + +if [[ $# -eq 2 ]]; then + REFERENCE=$2 + SHA=$(git rev-parse --short=8 ${REFERENCE}) +elif [[ -z ${PACKAGE_IMAGE_TAG} ]]; then + echo "Either PACKAGE_IMAGE_TAG or REFERENCE must be specified" + assert_usage "$@" +fi + +: ${CURL:=curl} + +: ${PACKAGE_IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit"} +: ${PACKAGE_IMAGE_TAG=${SHA}-packaging} + +VERSION="$(get_version_from_image ${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG} ${SHA})" + +REPO="experimental" +if [[ ${VERSION/rc./} == ${VERSION} ]]; then + REPO="stable" +fi + +PACKAGE_CACHE=release-${VERSION}-${REPO} +REMOVE_PACKAGE_CACHE=no +if [ ! -d ${PACKAGE_CACHE} ]; then +echo "Fetching packages with SHA '${SHA}' as tag '${VERSION}' to ${PACKAGE_CACHE}" +${SCRIPTS_DIR}/pull-packages.sh \ + ${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG} \ + ${PACKAGE_CACHE} + REMOVE_PACKAGE_CACHE=yes +else + echo "Using existing package cache: ${PACKAGE_CACHE}" +fi + +ARTIFACTS_DIR=${PROJECT_ROOT}/${PACKAGE_CACHE} + +IMAGE_EPOCH=$(extract_info "IMAGE_EPOCH") +# Note we use the main branch for the kitmaker archive. +GIT_BRANCH=main +GIT_COMMIT=$(extract_info "GIT_COMMIT") +GIT_COMMIT_SHORT=$(extract_info "GIT_COMMIT_SHORT") +PACKAGE_VERSION=$(extract_info "PACKAGE_VERSION") + +tar -czvf ${PACKAGE_CACHE}.tar.gz ${PACKAGE_CACHE} + +if [[ ${REMOVE_PACKAGE_CACHE} == "yes" ]]; then + rm -rf ${PACKAGE_CACHE} +fi + +: ${PACKAGE_ARCHIVE_FOLDER=releases-testing} + +function upload_archive() { + local archive=$1 + local component=$2 + local version=$3 + + if [ ! -r "${archive}" ]; then + echo "ERROR: File not found or not readable: ${archive}" + exit 1 + fi + local sha1_checksum=$(sha1sum -b "${archive}" | awk '{ print $1 }') + + local upload_url="${ARTIFACTORY_REPO}/${PACKAGE_ARCHIVE_FOLDER}/${component}/${version}/$(basename ${archive})" + + local props=() + # Required KITMAKER properties: + props+=("component_name=${component}") + props+=("version=${version}") + props+=("changelist=${GIT_COMMIT_SHORT}") + props+=("branch=${GIT_BRANCH}") + props+=("source=https://gitlab.com/nvidia/container-toolkit/container-toolkit") + # Package properties: + props+=("package.epoch=${IMAGE_EPOCH}") + props+=("package.version=${PACKAGE_VERSION}") + props+=("package.commit=${GIT_COMMIT}") + + for var in "CI_PROJECT_ID" "CI_PIPELINE_ID" "CI_JOB_ID" "CI_JOB_URL" "CI_PROJECT_PATH"; do + if [ -n "${!var}" ]; then + optionally_add_property "${var}" "${!var}" + fi + done + local PROPS=$(join_by ";" "${props[@]}") + + echo "Uploading ${upload_url} from ${archive}" + echo -H "X-JFrog-Art-Api: REDACTED" \ + -H "X-Checksum-Sha1: ${sha1_checksum}" \ + ${archive:+-T ${archive}} -X PUT \ + "${upload_url};${PROPS}" + if ! ${CURL} -f \ + -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \ + -H "X-Checksum-Sha1: ${sha1_checksum}" \ + ${archive:+-T ${archive}} -X PUT \ + "${upload_url};${PROPS}" ; + then + echo "ERROR: upload file failed: ${archive}" + exit 1 + fi +} + +upload_archive "${PACKAGE_CACHE}.tar.gz" "nvidia_container_toolkit" "${VERSION}" + +echo "Removing ${PACKAGE_CACHE}.tar.gz" +rm -f "${PACKAGE_CACHE}.tar.gz" +