From 71fbe7a81203034d933600e28e31a12361295170 Mon Sep 17 00:00:00 2001 From: Jon Mayo Date: Tue, 8 Nov 2022 16:45:34 +0000 Subject: [PATCH] [ci] push package releases to artifactory --- .common-ci.yml | 13 +- .gitlab-ci.yml | 2 +- .nvidia-ci.yml | 127 +++++++++++++- build/container/Dockerfile.packaging | 2 + scripts/build-all-components.sh | 24 +-- scripts/build-packages.sh | 11 +- scripts/extract-packages.sh | 112 +++++++++++++ scripts/packages-sign-all.sh | 18 +- scripts/release-kitmaker-artifactory.sh | 211 ++++++++++++++++++++++++ scripts/release-packages-artifactory.sh | 189 +++++++++++++++++++++ scripts/utils.sh | 36 ++++ 11 files changed, 715 insertions(+), 30 deletions(-) create mode 100755 scripts/extract-packages.sh create mode 100755 scripts/release-kitmaker-artifactory.sh create mode 100755 scripts/release-packages-artifactory.sh create mode 100644 scripts/utils.sh diff --git a/.common-ci.yml b/.common-ci.yml index 50172157..8602fd1c 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -47,6 +47,7 @@ stages: - !reference [.main-or-manual, rules] variables: DIST: amazonlinux2 + PACKAGE_REPO_TYPE: rpm .dist-centos7: rules: @@ -54,35 +55,41 @@ stages: variables: DIST: centos7 CVE_UPDATES: "cyrus-sasl-lib" + PACKAGE_REPO_TYPE: rpm .dist-centos8: variables: DIST: centos8 CVE_UPDATES: "cyrus-sasl-lib" + PACKAGE_REPO_TYPE: rpm .dist-debian10: rules: - !reference [.main-or-manual, rules] variables: DIST: debian10 + PACKAGE_REPO_TYPE: debian .dist-debian9: rules: - !reference [.main-or-manual, rules] variables: DIST: debian9 + PACKAGE_REPO_TYPE: debian .dist-fedora35: rules: - !reference [.main-or-manual, rules] variables: DIST: fedora35 + PACKAGE_REPO_TYPE: rpm .dist-opensuse-leap15.1: rules: - !reference [.main-or-manual, rules] variables: DIST: opensuse-leap15.1 + PACKAGE_REPO_TYPE: rpm .dist-ubi8: rules: @@ -90,17 +97,20 @@ stages: variables: DIST: ubi8 CVE_UPDATES: "cyrus-sasl-lib" + PACKAGE_REPO_TYPE: rpm .dist-ubuntu16.04: rules: - !reference [.main-or-manual, rules] variables: DIST: ubuntu16.04 + PACKAGE_REPO_TYPE: debian .dist-ubuntu18.04: variables: DIST: ubuntu18.04 CVE_UPDATES: "libsasl2-2 libsasl2-modules-db" + PACKAGE_REPO_TYPE: debian .dist-ubuntu20.04: rules: @@ -108,6 +118,7 @@ stages: variables: DIST: ubuntu20.04 CVE_UPDATES: "libsasl2-2 libsasl2-modules-db" + PACKAGE_REPO_TYPE: debian .dist-packaging: variables: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3b028ed6..b25ade65 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml index 475d832e..b97184d2 100644 --- a/.nvidia-ci.yml +++ b/.nvidia-ci.yml @@ -1,4 +1,4 @@ -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -35,6 +35,7 @@ variables: # Define the public staging registry STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging STAGING_VERSION: ${CI_COMMIT_SHORT_SHA} + ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative" .image-pull: stage: image-build @@ -92,6 +93,109 @@ image-packaging: - .dist-packaging - .image-pull +# Define the package release targets +release:packages:amazonlinux2-aarch64: + extends: + - .release:packages + - .dist-amazonlinux2 + - .arch-aarch64 + +release:packages:amazonlinux2-x86_64: + extends: + - .release:packages + - .dist-amazonlinux2 + - .arch-x86_64 + +release:packages:centos7-ppc64le: + extends: + - .release:packages + - .dist-centos7 + - .arch-ppc64le + +release:packages:centos7-x86_64: + extends: + - .release:packages + - .dist-centos7 + - .arch-x86_64 + +release:packages:centos8-aarch64: + extends: + - .release:packages + - .dist-centos8 + - .arch-aarch64 + +release:packages:centos8-ppc64le: + extends: + - .release:packages + - .dist-centos8 + - .arch-ppc64le + +release:packages:centos8-x86_64: + extends: + - .release:packages + - .dist-centos8 + - .arch-x86_64 + +release:packages:debian10-amd64: + extends: + - .release:packages + - .dist-debian10 + - .arch-amd64 + +release:packages:debian9-amd64: + extends: + - .release:packages + - .dist-debian9 + - .arch-amd64 + +release:packages:fedora35-aarch64: + extends: + - .release:packages + - .dist-fedora35 + - .arch-aarch64 + +release:packages:fedora35-x86_64: + extends: + - .release:packages + - .dist-fedora35 + - .arch-x86_64 + +release:packages:opensuse-leap15.1-x86_64: + extends: + - .release:packages + - .dist-opensuse-leap15.1 + - .arch-x86_64 + +release:packages:ubuntu16.04-amd64: + extends: + - .release:packages + - .dist-ubuntu16.04 + - .arch-amd64 + +release:packages:ubuntu16.04-ppc64le: + extends: + - .release:packages + - .dist-ubuntu16.04 + - .arch-ppc64le + +release:packages:ubuntu18.04-amd64: + extends: + - .release:packages + - .dist-ubuntu18.04 + - .arch-amd64 + +release:packages:ubuntu18.04-arm64: + extends: + - .release:packages + - .dist-ubuntu18.04 + - .arch-arm64 + +release:packages:ubuntu18.04-ppc64le: + extends: + - .release:packages + - .dist-ubuntu18.04 + - .arch-ppc64le + # We skip the integration tests for the internal CI: .integration: stage: test @@ -202,6 +306,27 @@ scan-ubi8-arm64: OUT_REGISTRY: "${NGC_REGISTRY}" OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}" +.release:packages: + stage: release + needs: + - image-packaging + variables: + PACKAGE_REGISTRY: "${CI_REGISTRY}" + PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}" + PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}" + PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit" + PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging" + PACKAGE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-${PACKAGE_REPO_TYPE}-local" + KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/kitmaker" + script: + - !reference [.regctl-setup, before_script] + - apk add --no-cache bash + - regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}" + - ./scripts/extract-packages.sh "${PACKAGE_REGISTRY}/${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}" + # TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}" + - ./scripts/kitmakerize-packages.sh" "${DIST}-${ARCH}" + - ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}" + release:staging-ubuntu18.04: extends: - .release:staging diff --git a/build/container/Dockerfile.packaging b/build/container/Dockerfile.packaging index d5358236..6873de41 100644 --- a/build/container/Dockerfile.packaging +++ b/build/container/Dockerfile.packaging @@ -25,5 +25,7 @@ ARG ARTIFACTS_ROOT COPY ${ARTIFACTS_ROOT} /artifacts/packages/ WORKDIR /artifacts/packages +# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container +RUN find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' > /artifacts/manifest.txt RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE diff --git a/scripts/build-all-components.sh b/scripts/build-all-components.sh index 145aa043..79922c59 100755 --- a/scripts/build-all-components.sh +++ b/scripts/build-all-components.sh @@ -21,14 +21,14 @@ function assert_usage() { echo "Missing argument $1" - echo "$(basename ${BASH_SOURCE[0]}) TARGET" + echo "$(basename "${BASH_SOURCE[0]}") TARGET" exit 1 } set -e SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" -PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )" +PROJECT_ROOT="$( cd "${SCRIPTS_DIR}"/.. && pwd )" if [[ $# -ne 1 ]]; then assert_usage "TARGET" @@ -36,31 +36,31 @@ fi TARGET=$1 -: ${DIST_DIR:=${PROJECT_ROOT}/dist} +: "${DIST_DIR:=${PROJECT_ROOT}/dist}" export DIST_DIR echo "Building ${TARGET} for all packages to ${DIST_DIR}" -: ${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container} -: ${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}} -: ${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime} -: ${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker} +: "${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}" +: "${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}" +: "${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}" +: "${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}" -${SCRIPTS_DIR}/get-component-versions.sh +"${SCRIPTS_DIR}/get-component-versions.sh" # Build libnvidia-container -make -C ${LIBNVIDIA_CONTAINER_ROOT} -f mk/docker.mk ${TARGET} +make -C "${LIBNVIDIA_CONTAINER_ROOT}" -f mk/docker.mk "${TARGET}" -if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_VERSION} || -z ${LIBNVIDIA_CONTAINER_VERSION} ]]; then +if [[ -z "${NVIDIA_CONTAINER_TOOLKIT_VERSION}" || -z "${LIBNVIDIA_CONTAINER_VERSION}" ]]; then eval $(${SCRIPTS_DIR}/get-component-versions.sh) fi # Build nvidia-container-toolkit -make -C ${NVIDIA_CONTAINER_TOOLKIT_ROOT} \ +make -C "${NVIDIA_CONTAINER_TOOLKIT_ROOT}" \ LIBNVIDIA_CONTAINER_VERSION="${LIBNVIDIA_CONTAINER_VERSION}" \ LIBNVIDIA_CONTAINER_TAG="${LIBNVIDIA_CONTAINER_TAG}" \ - ${TARGET} + "${TARGET}" if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then # We set the TOOLKIT_VERSION, TOOLKIT_TAG for the nvidia-container-runtime and nvidia-docker targets diff --git a/scripts/build-packages.sh b/scripts/build-packages.sh index 254d85bc..8c5da31e 100755 --- a/scripts/build-packages.sh +++ b/scripts/build-packages.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -22,7 +22,6 @@ set -e SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" -PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )" # This list represents the distribution-architecture pairs that are actually published # to the relevant repositories. This targets forwarded to the build-all-components script @@ -54,7 +53,7 @@ else fi echo "Updating components" -${SCRIPTS_DIR}/update-components.sh +"${SCRIPTS_DIR}/update-components.sh" if [[ -n $(git status -s third_party) && ${ALLOW_LOCAL_COMPONENT_CHANGES} != "true" ]]; then echo "ERROR: Building with local component changes." echo "Commit pending changes or rerun with ALLOW_LOCAL_COMPONENT_CHANGES='true'" @@ -66,7 +65,7 @@ eval $(${SCRIPTS_DIR}/get-component-versions.sh) if [[ -n ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then echo "Allowing mismatched versions for release candidate " -: ${ALLOW_VERSION_MISMATCH:=true} +: "${ALLOW_VERSION_MISMATCH:=true}" fi if [[ "${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" != "${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}" ]]; then @@ -86,6 +85,6 @@ export LIBNVIDIA_CONTAINER_TAG export NVIDIA_CONTAINER_RUNTIME_VERSION export NVIDIA_DOCKER_VERSION -for target in ${targets[@]}; do - ${SCRIPTS_DIR}/build-all-components.sh ${target} +for target in "${targets[@]}"; do + "${SCRIPTS_DIR}/build-all-components.sh" "${target}" done diff --git a/scripts/extract-packages.sh b/scripts/extract-packages.sh new file mode 100755 index 00000000..6d3d6edb --- /dev/null +++ b/scripts/extract-packages.sh @@ -0,0 +1,112 @@ +#!/usr/bin/env bash + +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function assert_usage() { + echo "Incorrect arguments: $*" >&2 + echo "$(basename "${BASH_SOURCE[0]}") PACKAGE_IMAGE_NAME:PACKAGE_IMAGE_TAG DIST-ARCH" >&2 + echo -e "\\tPACKAGE_IMAGE: container image holding packages [e.g. registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit]" >&2 + echo -e "\\tPACKAGE_TAG: tag for container image holding packages. [e.g. 1a2b3c4-packaging]" >&2 + echo -e "\\tDIST: The distribution." >&2 + echo -e "\\tARCH: The architecture." >&2 + exit 1 +} + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" +PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )" + +if [[ $# -ne 2 ]]; then + assert_usage "$@" +fi + +PACKAGE_IMAGE=$1 +DISTARCH=$2 +DIST=${DISTARCH%-*} +ARCH=${DISTARCH#*-} + +if [[ -z "${DIST}" || -z "${ARCH}" ]]; then + echo "ERROR: Distro and Architecture must be specified." >&2 + assert_usage "$@" +fi + +# TODO: accept ARTIFACTS_DIR as a command-line argument +: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}" + +# For release-candidates we skip certain packages. +# For example, we don't release release candidates of nvidia-container-runtime and nvidia-docker2 +# since these only bump the nvidia-container-toolkit dependency. +function skip-for-release-candidate() { + if [[ "${VERSION/rc./}" == "${VERSION}" ]]; then + return 1 + fi + + local package_name=$1 + if [[ "${package_name/"nvidia-docker2"/}" != "${package_name}" ]]; then + return 0 + fi + if [[ "${package_name/"nvidia-container-runtime"/}" != "${package_name}" ]]; then + return 0 + fi + return 1 +} + +# extract-file copies a file from a specified image. +# If regctl is available this is used, otherwise a docker container is run and the file is copied from +# there. +function copy-file() { + local image=$1 + local path_in_image=$2 + local path_on_host=$3 + if command -v regctl; then + regctl image get-file "${image}" "${path_in_image}" "${path_on_host}" + else + # Note this will only work for destinations where the `path_on_host` is in `pwd` + docker run --rm \ + -v "$(pwd):$(pwd)" \ + -w "$(pwd)" \ + -u "$(id -u):$(id -g)" \ + --entrypoint="bash" \ + "${image}" \ + -c "cp ${path_in_image} ${path_on_host}" + fi +} + +eval $(${SCRIPTS_DIR}/get-component-versions.sh) + +# extract-all extracts all package for the specified dist-arch combination from the package image. +# The manifest.txt file in the image is used to detemine the applicable files for the combination. +# Files are extracted to ${ARTIFACTS_DIR}/artifacts/packages/${dist}/${arch} +function extract-all() { + local dist=$1 + local arch=$2 + + mkdir -p "${ARTIFACTS_DIR}" + copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt" + + # Extract every file for the specified dist-arch combiniation in MANIFEST.txt + grep "/${dist}/${arch}/" "${ARTIFACTS_DIR}/manifest.txt" | while read -r f ; do + package_name="$(basename "$f")" + # For release-candidates, we skip certain packages + if skip-for-release-candidate "${package_name}"; then + echo "Skipping $f for release-candidate ${VERSION}" + continue + fi + target="${ARTIFACTS_DIR}/packages/${dist}/${arch}/${package_name}" + mkdir -p "$(dirname "$target")" + copy-file "${PACKAGE_IMAGE}" "${f}" "${target}" + done +} + +extract-all "${DIST}" "${ARCH}" diff --git a/scripts/packages-sign-all.sh b/scripts/packages-sign-all.sh index 11f0b0d9..ab61ea6a 100755 --- a/scripts/packages-sign-all.sh +++ b/scripts/packages-sign-all.sh @@ -9,7 +9,7 @@ set -x -e function deb-sign { local last_found - for r in ${*}; do + for r in "$@"; do if [ -f "./${r}" ]; then last_found=${r} fi @@ -27,12 +27,12 @@ function deb-sign { --no-emit-version \ --no-comments \ --personal-digest-preferences sha512 \ - --local-user ${GPG_LOCAL_USER} \ + --local-user "${GPG_LOCAL_USER}" \ > InRelease } function rpm-sign { - for r in ${*}; do + for r in "$@"; do if [ -f "./${r}" ]; then rpmsign --addsign --key-id A04EA552 --digest-algo=sha512 "${r}" fi @@ -42,7 +42,7 @@ function rpm-sign { --armor \ --no-emit-version \ --no-comments --personal-digest-preferences sha512 \ - --local-user ${GPG_LOCAL_USER} \ + --local-user "${GPG_LOCAL_USER}" \ repodata/repomd.xml } @@ -83,15 +83,15 @@ function sign() { return fi - cd ${dst} + cd "${dst}" if [[ -f "/etc/debian_version" ]]; then - [[ ${pkg_type} == "deb" ]] && deb-sign ${ALL_DEBS} + [[ "${pkg_type}" == "deb" ]] && deb-sign ${ALL_DEBS} else - [[ ${pkg_type} == "rpm" ]] && rpm-sign ${ALL_RPMS} + [[ "${pkg_type}" == "rpm" ]] && rpm-sign ${ALL_RPMS} fi cd - } -for target in ${TARGETS[@]}; do - sign ${target} $(pwd) +for target in "${TARGETS[@]}"; do + sign "${target}" "$(pwd)" done diff --git a/scripts/release-kitmaker-artifactory.sh b/scripts/release-kitmaker-artifactory.sh new file mode 100755 index 00000000..d96f0780 --- /dev/null +++ b/scripts/release-kitmaker-artifactory.sh @@ -0,0 +1,211 @@ +#!/usr/bin/env bash + +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +function assert_usage() { +cat >&2 << EOF +Incorrect arguments: $* +$(basename "${BASH_SOURCE[0]}") DIST-ARCH ARTIFACTORY_URL + DIST: The distribution. + ARCH: The architecture. + ARTIFACTORY_URL must contain repo path for package, including hostname. + +Environment Variables + ARTIFACTORY_TOKEN: must contain an auth token. [required] +EOF + exit 1 +} + +set -e + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" +PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )" +COMPONENT_NAME="nvidia-container-toolkit" + +if [[ $# -ne 1 ]]; then + assert_usage "$@" +fi + +source "${SCRIPTS_DIR}"/utils.sh + +DISTARCH=$1 +DIST=${DISTARCH%-*} +ARCH=${DISTARCH#*-} +ARTIFACTORY_URL=$2 + +if [[ -z "${DIST}" || -z "${ARCH}" ]]; then + echo "ERROR: Distro and Architecture must be specified." >&2 + assert_usage "$@" +fi + +# TODO: accept ARTIFACTS_DIR as a command-line argument +: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}" + +if [[ ! -d "${ARTIFACTS_DIR}" ]]; then + echo "ERROR: ARTIFACTS_DIR does not exist." >&2 + assert_usage "$@" +fi + +if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then + echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2 + assert_usage "$@" +fi + +# TODO: accept KITMACKER_DIR as a command-line argument +: "${KITMAKER_DIR="${PROJECT_ROOT}/artifacts/kitmaker"}" + +eval $(${SCRIPTS_DIR}/get-component-versions.sh) + +# Returns the key=value property if the value isn't empty +# Prepends with ";" if needed +set_prop_value() { + local key=$1 + local value=$2 + if [ -n "${value}" ]; then + if [ -z "${PROPS}" ]; then + echo "${key}=${value}" + else + echo ";${key}=${value}" + fi + fi +} + +process_props() { + local dist=$1 + local arch=$2 + + PROPS+=$(set_prop_value "component_name" "${COMPONENT_NAME}") + PROPS+=$(set_prop_value "version" "${VERSION}") + PROPS+=$(set_prop_value "os" "${dist}") + PROPS+=$(set_prop_value "arch" "${arch}") + PROPS+=$(set_prop_value "platform" "${dist}-${arch}") + # TODO: Use `git describe` to get this information if it's not available. + PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}") + PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}") + + # Gitlab variables to expose + for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do + if [ -n "${!var}" ]; then + PROPS+=$(set_prop_value "${var}" "${!var}") + fi + done +} + +## NOT USED: +## can substitute this function place of upload_file to modify properties of +## existing file instead of uploading files. +# Sets the properties on a path +# Relies on global variables: ARTIFACTORY_TOKEN, ARTIFACTORY_URL +set_props() { + local dist="$1" + local arch="$2" + local kitmakerfilename="$3" + + # extract the Artifactory hostname + artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}') + local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/${kitmakerfilename}" + + local PROPS + process_props "${DIST}" "${ARCH}" + + echo "Setting ${image_path} with properties: ${PROPS}" + if ! ${CURL} -fs -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \ + -X PUT \ + "https://${artifactory_host}/artifactory/api/storage/${image_path}?properties=${PROPS}&recursive=0" ; then + echo "ERROR: set props failed: ${image_path}" + exit 1 + fi +} + +# Uploads file to ARTIFACTS_DIR/// +# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_URL +upload_file() { + local dist=$1 + local arch=$2 + local file=$3 + + # extract the Artifactory hostname + artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}') + # get base part of the ARTIFACTORY_URL without hostname + local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}" + + local PROPS + process_props "${dist}" "${arch}" + + if [ ! -r "${file}" ]; then + echo "ERROR: File not found or not readable: ${file}" + exit 1 + fi + + # Collect sum + SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }') + + echo "Uploading ${image_path} from ${file}" + if ! ${CURL} -f \ + -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \ + -H "X-Checksum-Sha1: ${SHA1_SUM}" \ + ${file:+-T ${file}} -X PUT \ + "https://${artifactory_host}/artifactory/${image_path};${PROPS}" ; + then + echo "ERROR: upload file failed: ${file}" + exit 1 + fi +} + +function push-kitmaker-artifactory() { + local dist=$1 + local arch=$2 + local archive=$3 + + upload_file "${dist}" "${arch}" "${archive}" +} + +# kitmakerize-distro creates a tar.gz archive for the specified dist-arch combination. +# The archive is created at ${KITMAKER_DIR}/${name}.tar.gz (where ${name} is the third positional argument) +function kitmakerize-distro() { + local dist="$1" + local arch="$2" + local archive="$3" + + local name=$(basename "${archive%%.tar.gz}") + ## Copy packages into directory layout for .tar.gz + # TODO: make scratch_dir configurable + local scratch_dir="$(dirname ${archive})/.scratch/${name}" + local packages_dir="${scratch_dir}/.packages/" + + mkdir -p "${packages_dir}" + + # Copy the extracted files to the .packages directory so that a kitmaker file can be created. + source="${ARTIFACTS_DIR}/packages/${dist}/${arch}" + cp -r "${source}/"* "${packages_dir}/" + + ## Tar up the directory structure created above + tar zcvf "${archive}" -C "${scratch_dir}/.." "${name}" + echo "Created: ${archive}" + ls -l "${archive}" + echo "With contents:" + tar -tzvf "${archive}" + echo "" + + # Clean up the scratch directories: + rm -f "${scratch_dir}/.packages/"* + rmdir "${scratch_dir}/.packages" + rmdir "${scratch_dir}" +} + +kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" +kitmaker_archive="${KITMAKER_DIR}/${kitmaker_name}.tar.gz" +kitmakerize-distro "${DIST}" "${ARCH}" "${kitmaker_archive}" +push-kitmaker-artifactory "${DIST}" "${ARCH}" "${kitmaker_archive}" diff --git a/scripts/release-packages-artifactory.sh b/scripts/release-packages-artifactory.sh new file mode 100755 index 00000000..598b07cd --- /dev/null +++ b/scripts/release-packages-artifactory.sh @@ -0,0 +1,189 @@ +#!/usr/bin/env bash + +# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Dependencies: +# regctl +# + +function assert_usage() { +cat >&2 << EOF +Incorrect arguments: $* +$(basename "${BASH_SOURCE[0]}") DIST-ARCH + DIST: The distribution. + ARCH: The architecture. + +Environment Variables + ARTIFACTORY_TOKEN: must contain an auth token. [required] + LIB_TAG: optional package tag. + CI_COMMIT_REF_NAME: provided by CI/CD system. + CI_COMMIT_SHA: provided by CI/CD system. +EOF + exit 1 +} + +SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )" +PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )" + +source "${SCRIPTS_DIR}"/utils.sh + +if [[ $# -ne 1 ]]; then + assert_usage "$@" +fi + +DISTARCH=$1 +ARTIFACTORY_PATH=$2 +DIST=${DISTARCH%-*} +ARCH=${DISTARCH#*-} + +CURL=${CURL:-curl} + +if [[ -z "${DIST}" || -z "${ARCH}" ]]; then + echo "ERROR: Distro and Architecture must be specified." >&2 + assert_usage "$@" +fi + +if [[ -z "${ARTIFACTORY_PATH}" ]]; then + echo "ERROR: Package repo must be specified." >&2 + assert_usage "$@" +fi + +if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then + echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2 + assert_usage "$@" +fi + +# TODO: accept PACKAGES_DIR as a command-line argument +: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}" +: "${PACKAGES_DIR="${ARTIFACTS_DIR}/packages"}" + +eval $(${SCRIPTS_DIR}/get-component-versions.sh) + +# Returns the key=value property if the value isn't empty +# Prepends with ";" if needed +set_prop_value() { + local key=$1 + local value=$2 + if [ -n "${value}" ]; then + if [ -z "${PROPS}" ]; then + echo "${key}=${value}" + else + echo ";${key}=${value}" + fi + fi +} + +process_props() { + local dist=$1 + local arch=$2 + local file=$3 + local component_name="${file%%.*}" + component_name="${component_name%-*}" + local pkg_type="$(package_type $dist)" + + ## Component owner is free to define these + # PROPS+=$(set_prop_value "version" "${VERSION}") + # PROPS+=$(set_prop_value "lws_version" "${LWS_VER}") + # PROPS+=$(set_prop_value "platform" "${DISTARCH}") + + # TODO: Use `git describe` to get this information if it's not available. + PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}") + PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}") + + # PROPS+=$(set_prop_value "category" "utils") + # PROPS+=$(set_prop_value "platform" "${DISTARCH}") + + # Gitlab variables to expose + for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do + if [ -n "${!var}" ]; then + PROPS+=$(set_prop_value "${var}" "${!var}") + fi + done + + # We also set the package-specific properties to allow this to be used for other artifactory repositories + PROPS+=$(set_prop_value "${pkg_type}.distribution" "${dist}") + PROPS+=$(set_prop_value "${pkg_type}.architecture" "${arch}") + PROPS+=$(set_prop_value "${pkg_type}.component" "${component_name}") +} + +# Uploads file ARTIFACTORY_PATH +# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_PATH +upload_file() { + local dist=$1 + local arch=$2 + local file=$3 + + # TODO: These should be set by envvars + local artifactory_host="urm.nvidia.com" + local artifactory_repo="$(get_artifactory_repository $dist)" + + if [ ! -r "${file}" ]; then + echo "ERROR: File not found or not readable: ${file}" + exit 1 + fi + + local PROPS + process_props "${dist}" "${arch}" "${file}" + + # Collect sum + SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }') + + url="https://${artifactory_host}/artifactory/${artifactory_repo}/${dist}/${arch}/$(basename "${file}")" + # NOTE: The URL to set the properties through the API is: + # "https://${artifactory_host}/artifactory/api/storage/${artifactory_repo}/${dist}/${arch}/$(basename ${file})" + + echo "Uploading ${file} to ${url}" + if ! ${CURL} -f \ + -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \ + -H "X-Checksum-Sha1: ${SHA1_SUM}" \ + ${file:+-T ${file}} -X PUT \ + "${url};${PROPS}" ; + then + echo "ERROR: upload file failed: ${file}" + exit 1 + fi +} + +function push-artifactory() { + local dist="$1" + local arch="$2" + + source="${ARTIFACTS_DIR}/packages/${dist}/${arch}" + + find "${source}" -maxdepth 1 | while read -r f ; do + upload_file "$dist" "$arch" "$f" + done +} + +# TODO: use this to adapt as a general purpose command-line tool +# case "${COMMAND}" in +# set) +# set_props +# ;; +# upload) +# if [ -z "${UPLOAD_FILE}" ]; then +# echo "ERROR: Upload package filename must be set using -f" +# usage +# fi +# +# upload_file +# ;; +# *) +# echo "ERROR: Invalid command ${COMMAND}" +# usage +# ;; +# esac + +push-artifactory "${DIST}" "${ARCH}" diff --git a/scripts/utils.sh b/scripts/utils.sh new file mode 100644 index 00000000..32971191 --- /dev/null +++ b/scripts/utils.sh @@ -0,0 +1,36 @@ + +# package_type returns the packaging type (deb or rpm) for the specfied distribution. +# An error is returned if the ditribution is unsupported. +function package_type() { + local pkg_type + case ${1} in + amazonlinux*) pkg_type=rpm + ;; + centos*) pkg_type=rpm + ;; + debian*) pkg_type=deb + ;; + fedora*) pkg_type=rpm + ;; + opensuse-leap*) pkg_type=rpm + ;; + ubuntu*) pkg_type=deb + ;; + *) exit 1 + ;; + esac + echo "${pkg_type}" +} + +function get_artifactory_repository() { + local pkg_type=$(package_type $1) + + case ${pkg_type} in + deb) echo "sw-gpu-cloudnative-debian-local" + ;; + rpm) echo "sw-gpu-cloudnative-rpm-local" + ;; + *) echo "sw-gpu-cloudnative-generic-local" + ;; + esac +}