Merge branch 'CNT-4262/create-release-archives' into 'main'

Create and upload release archives to artifactory

See merge request nvidia/container-toolkit/container-toolkit!386

.nvidia-ci.yml

@@ -37,6 +37,7 @@ variables:
   STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
   ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
   KITMAKER_RELEASE_FOLDER: "kitmaker"
+  PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
 .image-pull:
   stage: image-build
@@ -223,6 +224,23 @@ release:packages:kitmaker:
   extends:
     - .release:packages
 
+release:archive:
+  extends:
+    - .release:external
+  needs:
+    - image-packaging
+  variables:
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+    PACKAGE_REGISTRY: "${CI_REGISTRY}"
+    PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
+    PACKAGE_ARCHIVE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${PACKAGE_ARCHIVE_RELEASE_FOLDER}"
+  script:
+    - apk add --no-cache bash git
+    - ./scripts/archive-packages.sh "${PACKAGE_ARCHIVE_ARTIFACTORY_REPO}"
+
 release:staging-ubuntu20.04:
   extends:
     - .release:staging

scripts/archive-packages.sh

@@ -0,0 +1,138 @@
+#!/usr/bin/env bash
+
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+function assert_usage() {
+    echo "Incorrect arguments: $*"
+    echo "$(basename ${BASH_SOURCE[0]}) ARTIFACTORY_REPO GIT_REFERENCE"
+    exit 1
+}
+
+set -e
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
+
+if [[ $# -le 1 ]]; then
+    assert_usage "$@"
+fi
+
+source "${SCRIPTS_DIR}"/utils.sh
+
+ARTIFACTORY_REPO=$1
+
+if [[ $# -eq 2 ]]; then
+    REFERENCE=$2
+    SHA=$(git rev-parse --short=8 ${REFERENCE})
+elif [[ -z ${PACKAGE_IMAGE_TAG} ]]; then
+    echo "Either PACKAGE_IMAGE_TAG or REFERENCE must be specified"
+    assert_usage "$@"
+fi
+
+: ${CURL:=curl}
+
+: ${PACKAGE_IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit"}
+: ${PACKAGE_IMAGE_TAG=${SHA}-packaging}
+
+VERSION="$(get_version_from_image ${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG} ${SHA})"
+
+REPO="experimental"
+if [[ ${VERSION/rc./} == ${VERSION} ]]; then
+    REPO="stable"
+fi
+
+PACKAGE_CACHE=release-${VERSION}-${REPO}
+REMOVE_PACKAGE_CACHE=no
+if [ ! -d ${PACKAGE_CACHE} ]; then
+echo "Fetching packages with SHA '${SHA}' as tag '${VERSION}' to ${PACKAGE_CACHE}"
+${SCRIPTS_DIR}/pull-packages.sh \
+    ${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG} \
+    ${PACKAGE_CACHE}
+    REMOVE_PACKAGE_CACHE=yes
+else
+    echo "Using existing package cache: ${PACKAGE_CACHE}"
+fi
+
+ARTIFACTS_DIR=${PROJECT_ROOT}/${PACKAGE_CACHE}
+
+IMAGE_EPOCH=$(extract_info "IMAGE_EPOCH")
+# Note we use the main branch for the kitmaker archive.
+GIT_BRANCH=main
+GIT_COMMIT=$(extract_info "GIT_COMMIT")
+GIT_COMMIT_SHORT=$(extract_info "GIT_COMMIT_SHORT")
+PACKAGE_VERSION=$(extract_info "PACKAGE_VERSION")
+
+tar -czvf ${PACKAGE_CACHE}.tar.gz ${PACKAGE_CACHE}
+
+if [[ ${REMOVE_PACKAGE_CACHE} == "yes" ]]; then
+    rm -rf ${PACKAGE_CACHE}
+fi
+
+: ${PACKAGE_ARCHIVE_FOLDER=releases-testing}
+
+function upload_archive() {
+    local archive=$1
+    local component=$2
+    local version=$3
+
+    if [ ! -r "${archive}" ]; then
+        echo "ERROR: File not found or not readable: ${archive}"
+        exit 1
+    fi
+    local sha1_checksum=$(sha1sum -b "${archive}" | awk '{ print $1 }')
+
+    local upload_url="${ARTIFACTORY_REPO}/${PACKAGE_ARCHIVE_FOLDER}/${component}/${version}/$(basename ${archive})"
+
+    local props=()
+    # Required KITMAKER properties:
+    props+=("component_name=${component}")
+    props+=("version=${version}")
+    props+=("changelist=${GIT_COMMIT_SHORT}")
+    props+=("branch=${GIT_BRANCH}")
+    props+=("source=https://gitlab.com/nvidia/container-toolkit/container-toolkit")
+    # Package properties:
+    props+=("package.epoch=${IMAGE_EPOCH}")
+    props+=("package.version=${PACKAGE_VERSION}")
+    props+=("package.commit=${GIT_COMMIT}")
+
+    for var in "CI_PROJECT_ID" "CI_PIPELINE_ID" "CI_JOB_ID" "CI_JOB_URL" "CI_PROJECT_PATH"; do
+        if [ -n "${!var}" ]; then
+            optionally_add_property "${var}" "${!var}"
+        fi
+    done
+    local PROPS=$(join_by ";" "${props[@]}")
+
+    echo "Uploading ${upload_url} from ${archive}"
+    echo -H "X-JFrog-Art-Api: REDACTED" \
+        -H "X-Checksum-Sha1: ${sha1_checksum}" \
+        ${archive:+-T ${archive}} -X PUT \
+        "${upload_url};${PROPS}"
+    if ! ${CURL} -f \
+        -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
+        -H "X-Checksum-Sha1: ${sha1_checksum}" \
+        ${archive:+-T ${archive}} -X PUT \
+        "${upload_url};${PROPS}" ;
+    then
+        echo "ERROR: upload file failed: ${archive}"
+        exit 1
+    fi
+}
+
+upload_archive "${PACKAGE_CACHE}.tar.gz" "nvidia_container_toolkit" "${VERSION}"
+
+echo "Removing ${PACKAGE_CACHE}.tar.gz"
+rm -f "${PACKAGE_CACHE}.tar.gz"
+

scripts/pull-packages.sh

@@ -55,4 +55,4 @@ docker run --rm \
     -u $(id -u):$(id -g) \
     --entrypoint="bash" \
         ${IMAGE} \
-        -c "cp -R /artifacts/packages/* ${DIST_DIR}"
+        -c "cp --preserve=timestamps -R /artifacts/* ${DIST_DIR}"

scripts/release-kitmaker-artifactory.sh

@@ -67,13 +67,6 @@ fi
 
 KITMAKER_SCRATCH="${KITMAKER_DIR}/.scratch"
 
-# extract_info extracts the value of the specified variable from the manifest.txt file.
-function extract_info() {
-    local variable=$1
-    local value=$(cat "${ARTIFACTS_DIR}/manifest.txt" | grep "#${variable}=" | sed -e "s/#${variable}=//" | tr -d '\r')
-    echo $value
-}
-
 IMAGE_EPOCH=$(extract_info "IMAGE_EPOCH")
 # Note we use the main branch for the kitmaker archive.
 GIT_BRANCH=main
@@ -129,8 +122,6 @@ function create_archive() {
     rmdir "${scratch_dir}"
 }
 
-function join_by { local IFS="$1"; shift; echo "$*"; }
-
 function optionally_add_property() {
     local property=$1
     local value=$2

scripts/release-packages.sh

@@ -31,6 +31,8 @@ if [[ $# -lt 1 || $# -gt 2 ]]; then
     assert_usage $*
 fi
 
+source "${SCRIPTS_DIR}"/utils.sh
+
 PACKAGE_REPO_ROOT=$1
 if [[ ! -d ${PACKAGE_REPO_ROOT} ]]; then
     echo "The specified PACKAGE_REPO_ROOT '${PACKAGE_REPO_ROOT}' must exist"
@@ -42,21 +44,20 @@ if [[ $# -ge 2 ]]; then
     REFERENCE=$2
 fi
 
-eval $(${SCRIPTS_DIR}/get-component-versions.sh)
-
-TAG=v"${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}"
 SHA=$(git rev-parse --short=8 ${REFERENCE})
+IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit"
+IMAGE_TAG=${SHA}-packaging
+
+VERSION="$(get_version_from_image ${IMAGE_NAME}:${IMAGE_TAG} ${SHA})"
 
 REPO="experimental"
-if [[ ${TAG/rc./} == ${TAG} ]]; then
+if [[ ${VERSION/rc./} == ${VERSION} ]]; then
     REPO="stable"
 fi
 
-PACKAGE_CACHE=release-${TAG}-${REPO}
+PACKAGE_CACHE=release-${VERSION}-${REPO}
 
-echo "Fetching packages with SHA '${SHA}' as tag '${TAG}' to ${PACKAGE_CACHE}"
+echo "Fetching packages with SHA '${SHA}' as tag '${VERSION}' to ${PACKAGE_CACHE}"
-IMAGE_NAME="registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit"
-IMAGE_TAG=${SHA}-packaging
 ${SCRIPTS_DIR}/pull-packages.sh \
     ${IMAGE_NAME}:${IMAGE_TAG} \
     ${PACKAGE_CACHE}
@@ -189,7 +190,7 @@ git -C ${PACKAGE_REPO_ROOT} reset --hard ${UPSTREAM_REFERENCE}
 git -C ${PACKAGE_REPO_ROOT} clean -fdx ${REPO}
 
 for target in ${targets[@]}; do
-    sync ${target} ${PACKAGE_CACHE} ${PACKAGE_REPO_ROOT}/${REPO}
+    sync ${target} ${PACKAGE_CACHE}/packages ${PACKAGE_REPO_ROOT}/${REPO}
 done
 
 git -C ${PACKAGE_REPO_ROOT} add ${REPO}
@@ -197,7 +198,7 @@ git -C ${PACKAGE_REPO_ROOT} add ${REPO}
 if [[ ${REPO} == "stable" ]]; then
 # Stable release
 git -C ${PACKAGE_REPO_ROOT} commit -s -F- <<EOF
-Add packages for NVIDIA Container Toolkit ${TAG} release
+Add packages for NVIDIA Container Toolkit ${VERSION} release
 
 These include:
 * libnvidia-container* ${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}
@@ -208,7 +209,7 @@ EOF
 else
 # Experimental / release candidate release
 git -C ${PACKAGE_REPO_ROOT} commit -s -F- <<EOF
-Add packages for NVIDIA Container Toolkit ${TAG} ${REPO} release
+Add packages for NVIDIA Container Toolkit ${VERSION} ${REPO} release
 
 These include:
 * libnvidia-container* ${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}
@@ -246,12 +247,12 @@ function sign() {
 sign deb
 
 git -C ${PACKAGE_REPO_ROOT} add ${REPO}
-git -C ${PACKAGE_REPO_ROOT} commit -s -m "TOFIX: Sign deb packages for ${TAG} in ${REPO}"
+git -C ${PACKAGE_REPO_ROOT} commit -s -m "TOFIX: Sign deb packages for ${VERSION} in ${REPO}"
 
 sign rpm
 
 git -C ${PACKAGE_REPO_ROOT} add ${REPO}
-git -C ${PACKAGE_REPO_ROOT} commit -s -m "TOFIX: Sign rpm packages for ${TAG} in ${REPO}"
+git -C ${PACKAGE_REPO_ROOT} commit -s -m "TOFIX: Sign rpm packages for ${VERSION} in ${REPO}"
 
 echo "To publish changes, go to ${PACKAGE_REPO_ROOT} and run: "
 echo "   git rebase -i ${UPSTREAM_REFERENCE}"

scripts/utils.sh

@@ -1,3 +1,17 @@
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
 
 # package_type returns the packaging type (deb or rpm) for the specfied distribution.
 # An error is returned if the ditribution is unsupported.
@@ -53,3 +67,48 @@ function get_package_target() {
         ;;
     esac
 }
+
+# extract-file copies a file from a specified image.
+# If regctl is available this is used, otherwise a docker container is run and the file is copied from
+# there.
+function copy_file() {
+    local image=$1
+    local path_in_image=$2
+    local path_on_host=$3
+    if command -v regctl > /dev/null; then
+        regctl image get-file "${image}" "${path_in_image}" "${path_on_host}"
+    else
+        # Note this will only work for destinations where the `path_on_host` is in `pwd`
+        docker run --rm \
+        -v "$(pwd):$(pwd)" \
+        -w "$(pwd)" \
+        -u "$(id -u):$(id -g)" \
+        --entrypoint="bash" \
+            "${image}" \
+            -c "cp ${path_in_image} ${path_on_host}"
+    fi
+}
+
+# extract_info extracts the value of the specified variable from the manifest.txt file.
+function extract_from_manifest() {
+    local variable=$1
+    local manifest=$2
+    local value=$(cat ${manifest} | grep "#${variable}=" | sed -e "s/#${variable}=//" | tr -d '\r')
+    echo $value
+}
+
+# extract_info extracts the value of the specified variable from the manifest.txt file.
+function extract_info() {
+    extract_from_manifest $1 "${ARTIFACTS_DIR}/manifest.txt"
+}
+
+function get_version_from_image() {
+    local image=$1
+    local manifest="manifest-${2}.txt"
+    copy_file ${image} "/artifacts/manifest.txt" ${manifest}
+    version=$(extract_from_manifest "PACKAGE_VERSION" ${manifest})
+    echo "v${version/\~/-}"
+    rm -f ${manifest}
+}
+
+function join_by { local IFS="$1"; shift; echo "$*"; }