diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index 366814b5..7260c114 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -37,6 +37,7 @@ variables: STAGING_VERSION: ${CI_COMMIT_SHORT_SHA} ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative" KITMAKER_RELEASE_FOLDER: "kitmaker" + PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases" .image-pull: stage: image-build @@ -216,6 +217,23 @@ release:packages:kitmaker: extends: - .release:packages +release:archive: + extends: + - .release:external + needs: + - image-packaging + variables: + VERSION: "${CI_COMMIT_SHORT_SHA}" + PACKAGE_REGISTRY: "${CI_REGISTRY}" + PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}" + PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}" + PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" + PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging" + PACKAGE_ARCHIVE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${PACKAGE_ARCHIVE_RELEASE_FOLDER}" + script: + - apk add --no-cache bash git + - ./scripts/archive-packages.sh "${PACKAGE_ARCHIVE_ARTIFACTORY_REPO}" + release:staging-ubuntu20.04: extends: - .release:staging diff --git a/scripts/archive-packages.sh b/scripts/archive-packages.sh new file mode 100755 index 00000000..5cbb5585 --- /dev/null +++ b/scripts/archive-packages.sh @@ -0,0 +1,138 @@ +#!/usr/bin/env bash + +# Copyright (c) NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +function assert_usage() { + echo "Incorrect arguments: $*" + echo "$(basename ${BASH_SOURCE[0]}) ARTIFACTORY_REPO GIT_REFERENCE" + exit 1 +} + +set -e + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" +PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )" + +if [[ $# -le 1 ]]; then + assert_usage "$@" +fi + +source "${SCRIPTS_DIR}"/utils.sh + +ARTIFACTORY_REPO=$1 + +if [[ $# -eq 2 ]]; then + REFERENCE=$2 + SHA=$(git rev-parse --short=8 ${REFERENCE}) +elif [[ -z ${PACKAGE_IMAGE_TAG} ]]; then + echo "Either PACKAGE_IMAGE_TAG or REFERENCE must be specified" + assert_usage "$@" +fi + +: ${CURL:=curl} + +: ${PACKAGE_IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit"} +: ${PACKAGE_IMAGE_TAG=${SHA}-packaging} + +VERSION="$(get_version_from_image ${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG} ${SHA})" + +REPO="experimental" +if [[ ${VERSION/rc./} == ${VERSION} ]]; then + REPO="stable" +fi + +PACKAGE_CACHE=release-${VERSION}-${REPO} +REMOVE_PACKAGE_CACHE=no +if [ ! -d ${PACKAGE_CACHE} ]; then +echo "Fetching packages with SHA '${SHA}' as tag '${VERSION}' to ${PACKAGE_CACHE}" +${SCRIPTS_DIR}/pull-packages.sh \ + ${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG} \ + ${PACKAGE_CACHE} + REMOVE_PACKAGE_CACHE=yes +else + echo "Using existing package cache: ${PACKAGE_CACHE}" +fi + +ARTIFACTS_DIR=${PROJECT_ROOT}/${PACKAGE_CACHE} + +IMAGE_EPOCH=$(extract_info "IMAGE_EPOCH") +# Note we use the main branch for the kitmaker archive. +GIT_BRANCH=main +GIT_COMMIT=$(extract_info "GIT_COMMIT") +GIT_COMMIT_SHORT=$(extract_info "GIT_COMMIT_SHORT") +PACKAGE_VERSION=$(extract_info "PACKAGE_VERSION") + +tar -czvf ${PACKAGE_CACHE}.tar.gz ${PACKAGE_CACHE} + +if [[ ${REMOVE_PACKAGE_CACHE} == "yes" ]]; then + rm -rf ${PACKAGE_CACHE} +fi + +: ${PACKAGE_ARCHIVE_FOLDER=releases-testing} + +function upload_archive() { + local archive=$1 + local component=$2 + local version=$3 + + if [ ! -r "${archive}" ]; then + echo "ERROR: File not found or not readable: ${archive}" + exit 1 + fi + local sha1_checksum=$(sha1sum -b "${archive}" | awk '{ print $1 }') + + local upload_url="${ARTIFACTORY_REPO}/${PACKAGE_ARCHIVE_FOLDER}/${component}/${version}/$(basename ${archive})" + + local props=() + # Required KITMAKER properties: + props+=("component_name=${component}") + props+=("version=${version}") + props+=("changelist=${GIT_COMMIT_SHORT}") + props+=("branch=${GIT_BRANCH}") + props+=("source=https://gitlab.com/nvidia/container-toolkit/container-toolkit") + # Package properties: + props+=("package.epoch=${IMAGE_EPOCH}") + props+=("package.version=${PACKAGE_VERSION}") + props+=("package.commit=${GIT_COMMIT}") + + for var in "CI_PROJECT_ID" "CI_PIPELINE_ID" "CI_JOB_ID" "CI_JOB_URL" "CI_PROJECT_PATH"; do + if [ -n "${!var}" ]; then + optionally_add_property "${var}" "${!var}" + fi + done + local PROPS=$(join_by ";" "${props[@]}") + + echo "Uploading ${upload_url} from ${archive}" + echo -H "X-JFrog-Art-Api: REDACTED" \ + -H "X-Checksum-Sha1: ${sha1_checksum}" \ + ${archive:+-T ${archive}} -X PUT \ + "${upload_url};${PROPS}" + if ! ${CURL} -f \ + -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \ + -H "X-Checksum-Sha1: ${sha1_checksum}" \ + ${archive:+-T ${archive}} -X PUT \ + "${upload_url};${PROPS}" ; + then + echo "ERROR: upload file failed: ${archive}" + exit 1 + fi +} + +upload_archive "${PACKAGE_CACHE}.tar.gz" "nvidia_container_toolkit" "${VERSION}" + +echo "Removing ${PACKAGE_CACHE}.tar.gz" +rm -f "${PACKAGE_CACHE}.tar.gz" +