From 8b197b27edd1aa09fbea8d798055ba82f10e5fc1 Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Mon, 23 Jan 2023 14:53:19 +0100
Subject: [PATCH] Rework the upload of archives to kitmaker.

This change simplifies how Kitmaker archives are constructed.

Currently only centos8 and ubuntu18.04 packages are included.

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 .nvidia-ci.yml                          | 115 +---------
 scripts/extract-packages.sh             |  34 ++-
 scripts/release-kitmaker-artifactory.sh | 285 ++++++++++++------------
 3 files changed, 168 insertions(+), 266 deletions(-)

diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml
index 23599e6a..d81c44dc 100644
--- a/.nvidia-ci.yml
+++ b/.nvidia-ci.yml
@@ -95,109 +95,6 @@ image-packaging:
     - .dist-packaging
     - .image-pull
 
-# Define the package release targets
-release:packages:amazonlinux2-aarch64:
-  extends:
-    - .release:packages
-    - .dist-amazonlinux2
-    - .arch-aarch64
-
-release:packages:amazonlinux2-x86_64:
-  extends:
-    - .release:packages
-    - .dist-amazonlinux2
-    - .arch-x86_64
-
-release:packages:centos7-ppc64le:
-  extends:
-    - .release:packages
-    - .dist-centos7
-    - .arch-ppc64le
-
-release:packages:centos7-x86_64:
-  extends:
-    - .release:packages
-    - .dist-centos7
-    - .arch-x86_64
-
-release:packages:centos8-aarch64:
-  extends:
-    - .release:packages
-    - .dist-centos8
-    - .arch-aarch64
-
-release:packages:centos8-ppc64le:
-  extends:
-    - .release:packages
-    - .dist-centos8
-    - .arch-ppc64le
-
-release:packages:centos8-x86_64:
-  extends:
-    - .release:packages
-    - .dist-centos8
-    - .arch-x86_64
-
-release:packages:debian10-amd64:
-  extends:
-    - .release:packages
-    - .dist-debian10
-    - .arch-amd64
-
-release:packages:debian9-amd64:
-  extends:
-    - .release:packages
-    - .dist-debian9
-    - .arch-amd64
-
-release:packages:fedora35-aarch64:
-  extends:
-    - .release:packages
-    - .dist-fedora35
-    - .arch-aarch64
-
-release:packages:fedora35-x86_64:
-  extends:
-    - .release:packages
-    - .dist-fedora35
-    - .arch-x86_64
-
-release:packages:opensuse-leap15.1-x86_64:
-  extends:
-    - .release:packages
-    - .dist-opensuse-leap15.1
-    - .arch-x86_64
-
-release:packages:ubuntu16.04-amd64:
-  extends:
-    - .release:packages
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-release:packages:ubuntu16.04-ppc64le:
-  extends:
-    - .release:packages
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
-release:packages:ubuntu18.04-amd64:
-  extends:
-    - .release:packages
-    - .dist-ubuntu18.04
-    - .arch-amd64
-
-release:packages:ubuntu18.04-arm64:
-  extends:
-    - .release:packages
-    - .dist-ubuntu18.04
-    - .arch-arm64
-
-release:packages:ubuntu18.04-ppc64le:
-  extends:
-    - .release:packages
-    - .dist-ubuntu18.04
-    - .arch-ppc64le
-
 # We skip the integration tests for the internal CI:
 .integration:
   stage: test
@@ -319,15 +216,19 @@ scan-ubi8-arm64:
     PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
     PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
     PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
-    PACKAGE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-${PACKAGE_REPO_TYPE}-local"
     KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
   script:
     - !reference [.regctl-setup, before_script]
-    - apk add --no-cache bash
+    - apk add --no-cache bash git
     - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
-    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}"
+    - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
     # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
-    - ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}"
+    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
+
+# Define the package release targets
+release:packages:kitmaker:
+  extends:
+    - .release:packages
 
 release:staging-ubuntu18.04:
   extends:
diff --git a/scripts/extract-packages.sh b/scripts/extract-packages.sh
index c4901fa4..9cc326b4 100755
--- a/scripts/extract-packages.sh
+++ b/scripts/extract-packages.sh
@@ -16,30 +16,22 @@
 
 function assert_usage() {
     echo "Incorrect arguments: $*" >&2
-    echo "$(basename "${BASH_SOURCE[0]}") PACKAGE_IMAGE_NAME:PACKAGE_IMAGE_TAG DIST-ARCH" >&2
+    echo "$(basename "${BASH_SOURCE[0]}") PACKAGE_IMAGE_NAME:PACKAGE_IMAGE_TAG" >&2
     echo -e "\\tPACKAGE_IMAGE: container image holding packages [e.g. registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit]" >&2
     echo -e "\\tPACKAGE_TAG: tag for container image holding packages. [e.g. 1a2b3c4-packaging]" >&2
-    echo -e "\\tDIST: The distribution." >&2
-    echo -e "\\tARCH: The architecture." >&2
     exit 1
 }
 
+set -e
+
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
 PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
 
-if [[ $# -ne 2 ]]; then
+if [[ $# -ne 1 ]]; then
     assert_usage "$@"
 fi
 
 PACKAGE_IMAGE=$1
-DISTARCH=$2
-DIST=${DISTARCH%-*}
-ARCH=${DISTARCH##*-}
-
-if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
-    echo "ERROR: Distro and Architecture must be specified." >&2
-    assert_usage "$@"
-fi
 
 # TODO: accept ARTIFACTS_DIR as a command-line argument
 : "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
@@ -85,30 +77,30 @@ function copy-file() {
 
 eval $(${SCRIPTS_DIR}/get-component-versions.sh)
 
-# extract-all extracts all package for the specified dist-arch combination from the package image.
+# extract-all extracts all package for the specified distribution from the package image.
 # The manifest.txt file in the image is used to detemine the applicable files for the combination.
 # Files are extracted to ${ARTIFACTS_DIR}/artifacts/packages/${dist}/${arch}
 function extract-all() {
     local dist=$1
-    local arch=$2
 
-    echo "Extracting packages for ${dist}-${arch} from ${PACKAGE_IMAGE}"
-
-    mkdir -p "${ARTIFACTS_DIR}"
-    copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt"
+    echo "Extracting packages for ${dist} from ${PACKAGE_IMAGE}"
 
     # Extract every file for the specified dist-arch combiniation in MANIFEST.txt
-    grep "/${dist}/${arch}/" "${ARTIFACTS_DIR}/manifest.txt" | while read -r f ; do
+    grep "/${dist}/" "${ARTIFACTS_DIR}/manifest.txt" | while read -r f ; do
         package_name="$(basename "$f")"
         # For release-candidates, we skip certain packages
         if skip-for-release-candidate "${package_name}"; then
             echo "Skipping $f for release-candidate ${VERSION}"
             continue
         fi
-        target="${ARTIFACTS_DIR}/packages/${dist}/${arch}/${package_name}"
+        target="${ARTIFACTS_DIR}/${f##/artifacts/}"
         mkdir -p "$(dirname "$target")"
         copy-file "${PACKAGE_IMAGE}" "${f}" "${target}"
     done
 }
 
-extract-all "${DIST}" "${ARCH}"
+mkdir -p "${ARTIFACTS_DIR}"
+copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt"
+
+extract-all ubuntu18.04
+extract-all centos8
diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh
index 2ac1ede9..1566383a 100755
--- a/scripts/release-kitmaker-artifactory.sh
+++ b/scripts/release-kitmaker-artifactory.sh
@@ -1,6 +1,6 @@
-#!/usr/bin/env bash
+#!/bin/bash
 
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,10 +17,8 @@
 function assert_usage() {
 cat >&2 << EOF
 Incorrect arguments: $*
-$(basename "${BASH_SOURCE[0]}") DIST-ARCH ARTIFACTORY_URL
-    DIST: The distribution.
-    ARCH: The architecture.
-    ARTIFACTORY_URL must contain repo path for package, including hostname.
+$(basename "${BASH_SOURCE[0]}") KITMAKER_ARTIFACTORY_REPO
+    KITMAKER_ARTIFACTORY_REPO must contain repo path for package, including hostname.
 
 Environment Variables
     ARTIFACTORY_TOKEN: must contain an auth token. [required]
@@ -32,26 +30,20 @@ set -e
 
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
 PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
-COMPONENT_NAME="nvidia-container-toolkit"
 
-if [[ $# -ne 2 ]]; then
+if [[ $# -ne 1 ]]; then
     assert_usage "$@"
 fi
 
 source "${SCRIPTS_DIR}"/utils.sh
 
-DISTARCH=$1
-DIST=${DISTARCH%-*}
-ARCH=${DISTARCH##*-}
-ARTIFACTORY_URL=$2
+# KITMAKER_ARTIFACTORY_REPO=https://urm.nvidia.com/artifactory/sw-gpu-cloudnative-generic-local/testing
+KITMAKER_ARTIFACTORY_REPO=$1
 
-CURL=${CURL:-curl}
-
-if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
-    echo "ERROR: Distro and Architecture must be specified." >&2
-    assert_usage "$@"
-fi
+: ${CURL:=curl}
 
+# ARTIFACTS_DIR represents the root of the artifacts (deb and rpm packages)
+# extracted from the packaging image.
 # TODO: accept ARTIFACTS_DIR as a command-line argument
 : "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
 
@@ -60,142 +52,68 @@ if [[ ! -d "${ARTIFACTS_DIR}" ]]; then
     assert_usage "$@"
 fi
 
+if [[ ! -f "${ARTIFACTS_DIR}/manifest.txt" ]]; then
+    echo "ERROR: Manifest file not found." >&2
+    assert_usage "$@"
+fi
+
 if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then
     echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2
     assert_usage "$@"
 fi
 
-# TODO: accept KITMACKER_DIR as a command-line argument
+# TODO: accept KITMAKER_DIR as a command-line argument
 : "${KITMAKER_DIR="${PROJECT_ROOT}/artifacts/kitmaker"}"
 
-eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+KITMAKER_SCRATCH="${KITMAKER_DIR}/.scratch"
 
-# Returns the key=value property if the value isn't empty
-# Prepends with ";" if needed
-set_prop_value() {
-    local key=$1
-    local value=$2
-    if [ -n "${value}" ]; then
-        if [ -z "${PROPS}" ]; then
-            echo "${key}=${value}"
-        else
-            echo ";${key}=${value}"
-        fi
-    fi
+# extract_info extracts the value of the specified variable from the manifest.txt file.
+function extract_info() {
+    local variable=$1
+    local value=$(cat "${ARTIFACTS_DIR}/manifest.txt" | grep "#${variable}" | sed -e "s/#${variable}=//" | tr -d '\r')
+    echo $value
 }
 
-process_props() {
-    local dist=$1
-    local arch=$2
+IMAGE_EPOCH=$(extract_info "IMAGE_EPOCH")
+GIT_BRANCH=$(extract_info "GIT_BRANCH")
+GIT_COMMIT=$(extract_info "GIT_COMMIT")
+VERSION=$(extract_info "VERSION")
 
-    PROPS+=$(set_prop_value "component_name" "${COMPONENT_NAME}")
-    PROPS+=$(set_prop_value "version" "${VERSION}")
-    PROPS+=$(set_prop_value "os" "${dist}")
-    PROPS+=$(set_prop_value "arch" "${arch}")
-    PROPS+=$(set_prop_value "platform" "${dist}-${arch}")
-    # TODO: Use `git describe` to get this information if it's not available.
-    PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}")
-    PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}")
 
-    # Gitlab variables to expose
-    for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do
-        if [ -n "${!var}" ]; then
-            PROPS+=$(set_prop_value "${var}" "${!var}")
-        fi
-    done
+# add_distro adds the specified component, os, and arch to the .package folder from which a kitmaker archive is generated.
+function add_distro() {
+    local component=$1
+    local os=$2
+    local arch=$3
 
-    echo "Applying properties: ${PROPS}"
-}
+    local package_dist=$4
+    local package_arch=$5
 
-## NOT USED:
-## can substitute this function place of upload_file to modify properties of
-## existing file instead of uploading files.
-# Sets the properties on a path
-# Relies on global variables: ARTIFACTORY_TOKEN, ARTIFACTORY_URL
-set_props() {
-    local dist="$1"
-    local arch="$2"
-    local kitmakerfilename="$3"
+    local name="${component}-${os}-${arch}"
 
-    # extract the Artifactory hostname
-    artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
-    local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/${kitmakerfilename}"
-
-    local PROPS
-    process_props "${DIST}" "${ARCH}"
-
-    echo "Setting ${image_path} with properties: ${PROPS}"
-    if ! ${CURL} -fs -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
-        -X PUT \
-        "https://${artifactory_host}/artifactory/api/storage/${image_path}?properties=${PROPS}&recursive=0" ; then
-        echo "ERROR: set props failed: ${image_path}"
-        exit 1
-    fi
-}
-
-# Uploads file to ARTIFACTS_DIR/<os>/<arch>/<filename>
-# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_URL
-upload_file() {
-    local dist=$1
-    local arch=$2
-    local file=$3
-
-    # extract the Artifactory hostname
-    artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
-    # get base part of the ARTIFACTORY_URL without hostname
-    local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/$(basename ${file})"
-
-    local PROPS
-    process_props "${dist}" "${arch}"
-
-    if [ ! -r "${file}" ]; then
-        echo "ERROR: File not found or not readable: ${file}"
-        exit 1
-    fi
-
-    # Collect sum
-    SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }')
-
-    echo "Uploading ${image_path} from ${file}"
-    if ! ${CURL} -f \
-        -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
-        -H "X-Checksum-Sha1: ${SHA1_SUM}" \
-        ${file:+-T ${file}} -X PUT \
-        "https://${artifactory_host}/${image_path};${PROPS}" ;
-        then
-        echo "ERROR: upload file failed: ${file}"
-        exit 1
-    fi
-}
-
-function push-kitmaker-artifactory() {
-    local dist=$1
-    local arch=$2
-    local archive=$3
-
-    upload_file "${dist}" "${arch}" "${archive}"
-}
-
-# kitmakerize-distro creates a tar.gz archive for the specified dist-arch combination.
-# The archive is created at ${KITMAKER_DIR}/${name}.tar.gz (where ${name} is the third positional argument)
-function kitmakerize-distro() {
-    local dist="$1"
-    local arch="$2"
-    local archive="$3"
-
-    local name=$(basename "${archive%%.tar.gz}")
-    ## Copy packages into directory layout for .tar.gz
-    # TODO: make scratch_dir configurable
-    local scratch_dir="$(dirname ${archive})/.scratch/${name}"
-    local packages_dir="${scratch_dir}/.packages/"
+    local scratch_dir="${KITMAKER_SCRATCH}/${name}"
+    local packages_dir="${scratch_dir}/.packages"
 
     mkdir -p "${packages_dir}"
 
     # Copy the extracted files to the .packages directory so that a kitmaker file can be created.
-    source="${ARTIFACTS_DIR}/packages/${dist}/${arch}"
+    source="${ARTIFACTS_DIR}/packages/${package_dist}/${package_arch}"
     cp -r "${source}/"* "${packages_dir}/"
+}
+
+# create_archive creates a kitmaker archive for the specified component, os, and arch.
+function create_archive() {
+    local component=$1
+    local os=$2
+    local arch=$3
+    local version=$4
+
+    local name="${component}-${os}-${arch}"
+    local archive="${KITMAKER_DIR}/${name}-${version}.tar.gz"
+
+    local scratch_dir="${KITMAKER_SCRATCH}/${name}"
+    local packages_dir="${scratch_dir}/.packages/"
 
-    ## Tar up the directory structure created above
     tar zcvf "${archive}" -C "${scratch_dir}/.." "${name}"
     echo "Created: ${archive}"
     ls -l "${archive}"
@@ -209,8 +127,99 @@ function kitmakerize-distro() {
     rmdir "${scratch_dir}"
 }
 
-: "${VERSION=$({NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION})}"
-kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${VERSION}"
-kitmaker_archive="${KITMAKER_DIR}/${kitmaker_name}.tar.gz"
-kitmakerize-distro "${DIST}" "${ARCH}" "${kitmaker_archive}"
-push-kitmaker-artifactory "${DIST}" "${ARCH}" "${kitmaker_archive}"
+function join_by { local IFS="$1"; shift; echo "$*"; }
+
+function optionally_add_property() {
+    local property=$1
+    local value=$2
+    if [[ -n "${value}" ]]; then
+        props+=("${property}=${value}")
+    fi
+}
+
+function upload_archive() {
+    local component=$1
+    local os=$2
+    local arch=$3
+    local version=$4
+
+    local package_builds=$(join_by , ${@:5})
+
+    local name="${component}-${os}-${arch}"
+    local archive="${KITMAKER_DIR}/${name}-${version}.tar.gz"
+
+    if [ ! -r "${archive}" ]; then
+        echo "ERROR: File not found or not readable: ${archive}"
+        exit 1
+    fi
+    local sha1_checksum=$(sha1sum -b "${archive}" | awk '{ print $1 }')
+
+    local upload_url="${KITMAKER_ARTIFACTORY_REPO}/${component}-${GIT_BRANCH}/default/$(basename ${archive})"
+
+    local props=()
+    # Required KITMAKER properties:
+    props+=("component_name=${component}")
+    props+=("version=${version}")
+    props+=("os=${os}")
+    props+=("arch=${arch}")
+    props+=("platform=${os}-${arch}")
+    # TODO: extract the GIT commit from the packaging image
+    props+=("changelist=${GIT_COMMIT}")
+    props+=("branch=${GIT_BRANCH}")
+    # Package properties:
+    props+=("package.epoch=${IMAGE_EPOCH}")
+    props+=("package.version=${VERSION}")
+    optionally_add_property "package.builds" "${package_builds}"
+
+    for var in "CI_PROJECT_ID" "CI_PIPELINE_ID" "CI_JOB_ID" "CI_JOB_URL" "CI_PROJECT_PATH"; do
+        if [ -n "${!var}" ]; then
+            optionally_add_property "${var}" "${!var}"
+        fi
+    done
+    local PROPS=$(join_by ";" "${props[@]}")
+
+    echo "Uploading ${upload_url} from ${file}"
+    if ! ${CURL} -f \
+        -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
+        -H "X-Checksum-Sha1: ${sha1_checksum}" \
+        ${archive:+-T ${archive}} -X PUT \
+        "${upload_url};${PROPS}" ;
+    then
+        echo "ERROR: upload file failed: ${archive}"
+        exit 1
+    fi
+}
+
+component="nvidia_container_toolkit"
+version="${VERSION%-rc.*}"
+version_suffix=$(date -r "${IMAGE_EPOCH}" '+%Y.%m.%d.%s' || date -d @"${IMAGE_EPOCH}" '+%Y.%m.%d.%s')
+kitmaker_version="${VERSION%-rc.*}.${version_suffix}"
+kitmaker_os="linux"
+
+# create_and_upload creates a kitmaker archive for the specified component, os, and arch and uploads it.
+function create_and_upload() {
+    local kitmaker_arch=$1
+    local builds=${@:2}
+
+    for build in ${builds}; do
+        local package_dist=$(echo ${build} | cut -d- -f1)
+        local package_arch=$(echo ${build} | cut -d- -f2)
+
+        add_distro "${component}" "${kitmaker_os}" "${kitmaker_arch}" "${package_dist}" "${package_arch}"
+    done
+
+    create_archive "${component}" "${kitmaker_os}" "${kitmaker_arch}" "${kitmaker_version}"
+    upload_archive "${component}" "${kitmaker_os}" "${kitmaker_arch}" "${kitmaker_version}" ${builds}
+}
+
+# Create archive for x86_64 linux distributions
+create_and_upload "x86_64" "ubuntu18.04-amd64" "centos8-x86_64"
+
+# Create archive for sbsa linux distributions
+create_and_upload "sbsa" "ubuntu18.04-arm64" "centos8-aarch64"
+# Create archive for aarch64 linux distributions
+# NOTE: From the perspective of the NVIDIA Container Toolkit aarch64 is just a duplicate of sbsa
+create_and_upload "aarch64" "ubuntu18.04-arm64" "centos8-aarch64"
+
+# Create archive for ppc64le linux distributions
+create_and_upload "ppc64le" "ubuntu18.04-ppc64le" "centos8-ppc64le"