From 30fb486e443c4558fc4d2c4802b71a56e487f12f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 14:27:58 +0100 Subject: [PATCH 01/11] Add basic logging Signed-off-by: Evan Lezar --- scripts/extract-packages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/extract-packages.sh b/scripts/extract-packages.sh index 6d3d6edb..234b80c8 100755 --- a/scripts/extract-packages.sh +++ b/scripts/extract-packages.sh @@ -92,6 +92,8 @@ function extract-all() { local dist=$1 local arch=$2 + echo "Extracting packages for ${dist}-${arch} from ${PACKAGE_IMAGE}" + mkdir -p "${ARTIFACTS_DIR}" copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt" From 83ce83239b1a946c87e5d4f7a52fa3b294aff9ce Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 14:33:48 +0100 Subject: [PATCH 02/11] Correct extract package image argument Signed-off-by: Evan Lezar --- .nvidia-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index b97184d2..9e6458c6 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -322,7 +322,7 @@ scan-ubi8-arm64: - !reference [.regctl-setup, before_script] - apk add --no-cache bash - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}" - - ./scripts/extract-packages.sh "${PACKAGE_REGISTRY}/${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}" + - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}" # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}" - ./scripts/kitmakerize-packages.sh" "${DIST}-${ARCH}" - ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}" From 6ef19d2925624192b412505646c4183bc420a228 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 14:42:56 +0100 Subject: [PATCH 03/11] Remove call to non-existant script Signed-off-by: Evan Lezar --- .nvidia-ci.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index 9e6458c6..17b69050 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -324,7 +324,6 @@ scan-ubi8-arm64: - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}" - ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}" # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}" - - ./scripts/kitmakerize-packages.sh" "${DIST}-${ARCH}" - ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}" release:staging-ubuntu18.04: From 7fd40632fea18f81d2fe5a0a13f1131125d04ba6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 14:43:17 +0100 Subject: [PATCH 04/11] Update regctl version The regctl image copy-file command was added in v0.4.5. Signed-off-by: Evan Lezar --- .common-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.common-ci.yml b/.common-ci.yml index 8602fd1c..c28bf25e 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -180,7 +180,7 @@ test-packaging: # Download the regctl binary for use in the release steps .regctl-setup: before_script: - - export REGCTL_VERSION=v0.3.10 + - export REGCTL_VERSION=v0.4.5 - apk add --no-cache curl - mkdir -p bin - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64 From 9db31f7506dd5b2e8845637853fcad12ea190935 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 14:50:04 +0100 Subject: [PATCH 05/11] Fix number of arguments for kitmaker release script Signed-off-by: Evan Lezar --- scripts/release-kitmaker-artifactory.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh index d96f0780..8d8834d0 100755 --- a/scripts/release-kitmaker-artifactory.sh +++ b/scripts/release-kitmaker-artifactory.sh @@ -34,7 +34,7 @@ SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )" COMPONENT_NAME="nvidia-container-toolkit" -if [[ $# -ne 1 ]]; then +if [[ $# -ne 2 ]]; then assert_usage "$@" fi From 08640a6f64d5e27d7f30adadba39fc78f2960d8c Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 14:56:09 +0100 Subject: [PATCH 06/11] Ensure CURL is set for kitmaker upload Signed-off-by: Evan Lezar --- scripts/release-kitmaker-artifactory.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh index 8d8834d0..7e1b24b1 100755 --- a/scripts/release-kitmaker-artifactory.sh +++ b/scripts/release-kitmaker-artifactory.sh @@ -45,6 +45,8 @@ DIST=${DISTARCH%-*} ARCH=${DISTARCH#*-} ARTIFACTORY_URL=$2 +CURL=${CURL:-curl} + if [[ -z "${DIST}" || -z "${ARCH}" ]]; then echo "ERROR: Distro and Architecture must be specified." >&2 assert_usage "$@" From 80f8c2a4185291cfa1adcecaa56016cdc96399f7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 15:14:08 +0100 Subject: [PATCH 07/11] Correct artifactory upload URL Signed-off-by: Evan Lezar --- scripts/release-kitmaker-artifactory.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh index 7e1b24b1..3d511cde 100755 --- a/scripts/release-kitmaker-artifactory.sh +++ b/scripts/release-kitmaker-artifactory.sh @@ -141,7 +141,7 @@ upload_file() { # extract the Artifactory hostname artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}') # get base part of the ARTIFACTORY_URL without hostname - local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}" + local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/$(basename ${file})" local PROPS process_props "${dist}" "${arch}" @@ -159,7 +159,7 @@ upload_file() { -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \ -H "X-Checksum-Sha1: ${SHA1_SUM}" \ ${file:+-T ${file}} -X PUT \ - "https://${artifactory_host}/artifactory/${image_path};${PROPS}" ; + "https://${artifactory_host}/${image_path};${PROPS}" ; then echo "ERROR: upload file failed: ${file}" exit 1 From c958817eefd6841dcdcc308b9fdba336d6108599 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 15:33:32 +0100 Subject: [PATCH 08/11] Log applied properties. Signed-off-by: Evan Lezar --- scripts/release-kitmaker-artifactory.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh index 3d511cde..a635b9f5 100755 --- a/scripts/release-kitmaker-artifactory.sh +++ b/scripts/release-kitmaker-artifactory.sh @@ -103,6 +103,8 @@ process_props() { PROPS+=$(set_prop_value "${var}" "${!var}") fi done + + echo "Applying properties: ${PROPS}" } ## NOT USED: From 75b11eb80a8ef3f7dff9fc7494ccfad7179364e2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 15:34:00 +0100 Subject: [PATCH 09/11] Use VERSION in kitmaker archive name Signed-off-by: Evan Lezar --- scripts/release-kitmaker-artifactory.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh index a635b9f5..4d433f28 100755 --- a/scripts/release-kitmaker-artifactory.sh +++ b/scripts/release-kitmaker-artifactory.sh @@ -209,7 +209,8 @@ function kitmakerize-distro() { rmdir "${scratch_dir}" } -kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" +: "${VERSION=$(git describe --tags --dirty)}" +kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${VERSION}" kitmaker_archive="${KITMAKER_DIR}/${kitmaker_name}.tar.gz" kitmakerize-distro "${DIST}" "${ARCH}" "${kitmaker_archive}" push-kitmaker-artifactory "${DIST}" "${ARCH}" "${kitmaker_archive}" From 7955bb1a84fbc2b80073d3029bb3a7e1d3c2cb87 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 15:52:27 +0100 Subject: [PATCH 10/11] Use short sha for kitmaker version Signed-off-by: Evan Lezar --- .nvidia-ci.yml | 1 + scripts/release-kitmaker-artifactory.sh | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index 17b69050..a2954b24 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -311,6 +311,7 @@ scan-ubi8-arm64: needs: - image-packaging variables: + VERSION: "${CI_COMMIT_SHORT_SHA}" PACKAGE_REGISTRY: "${CI_REGISTRY}" PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}" PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}" diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh index 4d433f28..5b5d7bf0 100755 --- a/scripts/release-kitmaker-artifactory.sh +++ b/scripts/release-kitmaker-artifactory.sh @@ -209,7 +209,7 @@ function kitmakerize-distro() { rmdir "${scratch_dir}" } -: "${VERSION=$(git describe --tags --dirty)}" +: "${VERSION=$({NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION})}" kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${VERSION}" kitmaker_archive="${KITMAKER_DIR}/${kitmaker_name}.tar.gz" kitmakerize-distro "${DIST}" "${ARCH}" "${kitmaker_archive}" From 74abea07e27ba2134554aa2fe29db70b14f14d75 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 10 Nov 2022 16:00:03 +0100 Subject: [PATCH 11/11] Add top-level variable to set kitmaker folder Signed-off-by: Evan Lezar --- .nvidia-ci.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index a2954b24..23599e6a 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -36,6 +36,8 @@ variables: STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging STAGING_VERSION: ${CI_COMMIT_SHORT_SHA} ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative" + # TODO: We should set the kitmaker release folder here once we have the end-to-end workflow set up + KITMAKER_RELEASE_FOLDER: "testing" .image-pull: stage: image-build @@ -318,7 +320,7 @@ scan-ubi8-arm64: PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging" PACKAGE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-${PACKAGE_REPO_TYPE}-local" - KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/kitmaker" + KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}" script: - !reference [.regctl-setup, before_script] - apk add --no-cache bash