UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2025-06-26 18:18:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

[ci] push package releases to artifactory

Browse Source
This commit is contained in:
Jon Mayo 2022-11-08 16:45:34 +00:00
parent 0bc09665a8
commit 71fbe7a812
11 changed files with 715 additions and 30 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
.common-ci.yml
View File

@ -1,4 +1,4 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -47,6 +47,7 @@ stages:
- !reference [.main-or-manual, rules]
variables:
DIST: amazonlinux2
PACKAGE_REPO_TYPE: rpm
.dist-centos7:
rules:
@ -54,35 +55,41 @@ stages:
variables:
DIST: centos7
CVE_UPDATES: "cyrus-sasl-lib"
PACKAGE_REPO_TYPE: rpm
.dist-centos8:
variables:
DIST: centos8
CVE_UPDATES: "cyrus-sasl-lib"
PACKAGE_REPO_TYPE: rpm
.dist-debian10:
rules:
- !reference [.main-or-manual, rules]
variables:
DIST: debian10
PACKAGE_REPO_TYPE: debian
.dist-debian9:
rules:
- !reference [.main-or-manual, rules]
variables:
DIST: debian9
PACKAGE_REPO_TYPE: debian
.dist-fedora35:
rules:
- !reference [.main-or-manual, rules]
variables:
DIST: fedora35
PACKAGE_REPO_TYPE: rpm
.dist-opensuse-leap15.1:
rules:
- !reference [.main-or-manual, rules]
variables:
DIST: opensuse-leap15.1
PACKAGE_REPO_TYPE: rpm
.dist-ubi8:
rules:
@ -90,17 +97,20 @@ stages:
variables:
DIST: ubi8
CVE_UPDATES: "cyrus-sasl-lib"
PACKAGE_REPO_TYPE: rpm
.dist-ubuntu16.04:
rules:
- !reference [.main-or-manual, rules]
variables:
DIST: ubuntu16.04
PACKAGE_REPO_TYPE: debian
.dist-ubuntu18.04:
variables:
DIST: ubuntu18.04
CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
PACKAGE_REPO_TYPE: debian
.dist-ubuntu20.04:
rules:
@ -108,6 +118,7 @@ stages:
variables:
DIST: ubuntu20.04
CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
PACKAGE_REPO_TYPE: debian
.dist-packaging:
variables:

2
.gitlab-ci.yml
View File

@ -1,4 +1,4 @@
# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.

127
.nvidia-ci.yml
View File

@ -1,4 +1,4 @@
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -35,6 +35,7 @@ variables:
# Define the public staging registry
STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
.image-pull:
stage: image-build
@ -92,6 +93,109 @@ image-packaging:
- .dist-packaging
- .image-pull
# Define the package release targets
release:packages:amazonlinux2-aarch64:
extends:
- .release:packages
- .dist-amazonlinux2
- .arch-aarch64
release:packages:amazonlinux2-x86_64:
extends:
- .release:packages
- .dist-amazonlinux2
- .arch-x86_64
release:packages:centos7-ppc64le:
extends:
- .release:packages
- .dist-centos7
- .arch-ppc64le
release:packages:centos7-x86_64:
extends:
- .release:packages
- .dist-centos7
- .arch-x86_64
release:packages:centos8-aarch64:
extends:
- .release:packages
- .dist-centos8
- .arch-aarch64
release:packages:centos8-ppc64le:
extends:
- .release:packages
- .dist-centos8
- .arch-ppc64le
release:packages:centos8-x86_64:
extends:
- .release:packages
- .dist-centos8
- .arch-x86_64
release:packages:debian10-amd64:
extends:
- .release:packages
- .dist-debian10
- .arch-amd64
release:packages:debian9-amd64:
extends:
- .release:packages
- .dist-debian9
- .arch-amd64
release:packages:fedora35-aarch64:
extends:
- .release:packages
- .dist-fedora35
- .arch-aarch64
release:packages:fedora35-x86_64:
extends:
- .release:packages
- .dist-fedora35
- .arch-x86_64
release:packages:opensuse-leap15.1-x86_64:
extends:
- .release:packages
- .dist-opensuse-leap15.1
- .arch-x86_64
release:packages:ubuntu16.04-amd64:
extends:
- .release:packages
- .dist-ubuntu16.04
- .arch-amd64
release:packages:ubuntu16.04-ppc64le:
extends:
- .release:packages
- .dist-ubuntu16.04
- .arch-ppc64le
release:packages:ubuntu18.04-amd64:
extends:
- .release:packages
- .dist-ubuntu18.04
- .arch-amd64
release:packages:ubuntu18.04-arm64:
extends:
- .release:packages
- .dist-ubuntu18.04
- .arch-arm64
release:packages:ubuntu18.04-ppc64le:
extends:
- .release:packages
- .dist-ubuntu18.04
- .arch-ppc64le
# We skip the integration tests for the internal CI:
.integration:
stage: test
@ -202,6 +306,27 @@ scan-ubi8-arm64:
OUT_REGISTRY: "${NGC_REGISTRY}"
OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
.release:packages:
stage: release
needs:
- image-packaging
variables:
PACKAGE_REGISTRY: "${CI_REGISTRY}"
PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
PACKAGE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-${PACKAGE_REPO_TYPE}-local"
KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/kitmaker"
script:
- !reference [.regctl-setup, before_script]
- apk add --no-cache bash
- regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
- ./scripts/extract-packages.sh "${PACKAGE_REGISTRY}/${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}"
# TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
- ./scripts/kitmakerize-packages.sh" "${DIST}-${ARCH}"
- ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}"
release:staging-ubuntu18.04:
extends:
- .release:staging

2
build/container/Dockerfile.packaging
View File

@ -25,5 +25,7 @@ ARG ARTIFACTS_ROOT
COPY ${ARTIFACTS_ROOT} /artifacts/packages/
WORKDIR /artifacts/packages
# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
RUN find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' > /artifacts/manifest.txt
RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

24
scripts/build-all-components.sh
View File

@ -21,14 +21,14 @@
function assert_usage() {
echo "Missing argument $1"
echo "$(basename ${BASH_SOURCE[0]}) TARGET"
echo "$(basename "${BASH_SOURCE[0]}") TARGET"
exit 1
}
set -e
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}"/.. && pwd )"
if [[ $# -ne 1 ]]; then
assert_usage "TARGET"
@ -36,31 +36,31 @@ fi
TARGET=$1
: ${DIST_DIR:=${PROJECT_ROOT}/dist}
: "${DIST_DIR:=${PROJECT_ROOT}/dist}"
export DIST_DIR
echo "Building ${TARGET} for all packages to ${DIST_DIR}"
: ${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}
: ${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}
: ${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}
: ${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}
: "${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}"
: "${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}"
: "${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}"
: "${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}"
${SCRIPTS_DIR}/get-component-versions.sh
"${SCRIPTS_DIR}/get-component-versions.sh"
# Build libnvidia-container
make -C ${LIBNVIDIA_CONTAINER_ROOT} -f mk/docker.mk ${TARGET}
make -C "${LIBNVIDIA_CONTAINER_ROOT}" -f mk/docker.mk "${TARGET}"
if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_VERSION} || -z ${LIBNVIDIA_CONTAINER_VERSION} ]]; then
if [[ -z "${NVIDIA_CONTAINER_TOOLKIT_VERSION}" || -z "${LIBNVIDIA_CONTAINER_VERSION}" ]]; then
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
fi
# Build nvidia-container-toolkit
make -C ${NVIDIA_CONTAINER_TOOLKIT_ROOT} \
make -C "${NVIDIA_CONTAINER_TOOLKIT_ROOT}" \
LIBNVIDIA_CONTAINER_VERSION="${LIBNVIDIA_CONTAINER_VERSION}" \
LIBNVIDIA_CONTAINER_TAG="${LIBNVIDIA_CONTAINER_TAG}" \
${TARGET}
"${TARGET}"
if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then
# We set the TOOLKIT_VERSION, TOOLKIT_TAG for the nvidia-container-runtime and nvidia-docker targets

11
scripts/build-packages.sh
View File

@ -1,6 +1,6 @@
#!/usr/bin/env bash
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -22,7 +22,6 @@
set -e
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
# This list represents the distribution-architecture pairs that are actually published
# to the relevant repositories. This targets forwarded to the build-all-components script
@ -54,7 +53,7 @@ else
fi
echo "Updating components"
${SCRIPTS_DIR}/update-components.sh
"${SCRIPTS_DIR}/update-components.sh"
if [[ -n $(git status -s third_party) && ${ALLOW_LOCAL_COMPONENT_CHANGES} != "true" ]]; then
echo "ERROR: Building with local component changes."
echo "Commit pending changes or rerun with ALLOW_LOCAL_COMPONENT_CHANGES='true'"
@ -66,7 +65,7 @@ eval $(${SCRIPTS_DIR}/get-component-versions.sh)
if [[ -n ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then
echo "Allowing mismatched versions for release candidate "
: ${ALLOW_VERSION_MISMATCH:=true}
: "${ALLOW_VERSION_MISMATCH:=true}"
fi
if [[ "${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" != "${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}" ]]; then
@ -86,6 +85,6 @@ export LIBNVIDIA_CONTAINER_TAG
export NVIDIA_CONTAINER_RUNTIME_VERSION
export NVIDIA_DOCKER_VERSION
for target in ${targets[@]}; do
${SCRIPTS_DIR}/build-all-components.sh ${target}
for target in "${targets[@]}"; do
"${SCRIPTS_DIR}/build-all-components.sh" "${target}"
done

112
scripts/extract-packages.sh Executable file
View File

@ -0,0 +1,112 @@
#!/usr/bin/env bash
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
function assert_usage() {
echo "Incorrect arguments: $*" >&2
echo "$(basename "${BASH_SOURCE[0]}") PACKAGE_IMAGE_NAME:PACKAGE_IMAGE_TAG DIST-ARCH" >&2
echo -e "\\tPACKAGE_IMAGE: container image holding packages [e.g. registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit]" >&2
echo -e "\\tPACKAGE_TAG: tag for container image holding packages. [e.g. 1a2b3c4-packaging]" >&2
echo -e "\\tDIST: The distribution." >&2
echo -e "\\tARCH: The architecture." >&2
exit 1
}
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
if [[ $# -ne 2 ]]; then
assert_usage "$@"
fi
PACKAGE_IMAGE=$1
DISTARCH=$2
DIST=${DISTARCH%-*}
ARCH=${DISTARCH#*-}
if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
echo "ERROR: Distro and Architecture must be specified." >&2
assert_usage "$@"
fi
# TODO: accept ARTIFACTS_DIR as a command-line argument
: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
# For release-candidates we skip certain packages.
# For example, we don't release release candidates of nvidia-container-runtime and nvidia-docker2
# since these only bump the nvidia-container-toolkit dependency.
function skip-for-release-candidate() {
if [[ "${VERSION/rc./}" == "${VERSION}" ]]; then
return 1
fi
local package_name=$1
if [[ "${package_name/"nvidia-docker2"/}" != "${package_name}" ]]; then
return 0
fi
if [[ "${package_name/"nvidia-container-runtime"/}" != "${package_name}" ]]; then
return 0
fi
return 1
}
# extract-file copies a file from a specified image.
# If regctl is available this is used, otherwise a docker container is run and the file is copied from
# there.
function copy-file() {
local image=$1
local path_in_image=$2
local path_on_host=$3
if command -v regctl; then
regctl image get-file "${image}" "${path_in_image}" "${path_on_host}"
else
# Note this will only work for destinations where the `path_on_host` is in `pwd`
docker run --rm \
-v "$(pwd):$(pwd)" \
-w "$(pwd)" \
-u "$(id -u):$(id -g)" \
--entrypoint="bash" \
"${image}" \
-c "cp ${path_in_image} ${path_on_host}"
fi
}
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
# extract-all extracts all package for the specified dist-arch combination from the package image.
# The manifest.txt file in the image is used to detemine the applicable files for the combination.
# Files are extracted to ${ARTIFACTS_DIR}/artifacts/packages/${dist}/${arch}
function extract-all() {
local dist=$1
local arch=$2
mkdir -p "${ARTIFACTS_DIR}"
copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt"
# Extract every file for the specified dist-arch combiniation in MANIFEST.txt
grep "/${dist}/${arch}/" "${ARTIFACTS_DIR}/manifest.txt" | while read -r f ; do
package_name="$(basename "$f")"
# For release-candidates, we skip certain packages
if skip-for-release-candidate "${package_name}"; then
echo "Skipping $f for release-candidate ${VERSION}"
continue
fi
target="${ARTIFACTS_DIR}/packages/${dist}/${arch}/${package_name}"
mkdir -p "$(dirname "$target")"
copy-file "${PACKAGE_IMAGE}" "${f}" "${target}"
done
}
extract-all "${DIST}" "${ARCH}"

18
scripts/packages-sign-all.sh
View File

@ -9,7 +9,7 @@ set -x -e
function deb-sign {
local last_found
for r in ${*}; do
for r in "$@"; do
if [ -f "./${r}" ]; then
last_found=${r}
fi
@ -27,12 +27,12 @@ function deb-sign {
--no-emit-version \
--no-comments \
--personal-digest-preferences sha512 \
--local-user ${GPG_LOCAL_USER} \
--local-user "${GPG_LOCAL_USER}" \
> InRelease
}
function rpm-sign {
for r in ${*}; do
for r in "$@"; do
if [ -f "./${r}" ]; then
rpmsign --addsign --key-id A04EA552 --digest-algo=sha512 "${r}"
fi
@ -42,7 +42,7 @@ function rpm-sign {
--armor \
--no-emit-version \
--no-comments --personal-digest-preferences sha512 \
--local-user ${GPG_LOCAL_USER} \
--local-user "${GPG_LOCAL_USER}" \
repodata/repomd.xml
}
@ -83,15 +83,15 @@ function sign() {
return
fi
cd ${dst}
cd "${dst}"
if [[ -f "/etc/debian_version" ]]; then
[[ ${pkg_type} == "deb" ]] && deb-sign ${ALL_DEBS}
[[ "${pkg_type}" == "deb" ]] && deb-sign ${ALL_DEBS}
else
[[ ${pkg_type} == "rpm" ]] && rpm-sign ${ALL_RPMS}
[[ "${pkg_type}" == "rpm" ]] && rpm-sign ${ALL_RPMS}
fi
cd -
}
for target in ${TARGETS[@]}; do
sign ${target} $(pwd)
for target in "${TARGETS[@]}"; do
sign "${target}" "$(pwd)"
done

211
scripts/release-kitmaker-artifactory.sh Executable file
View File

@ -0,0 +1,211 @@
#!/usr/bin/env bash
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
function assert_usage() {
cat >&2 << EOF
Incorrect arguments: $*
$(basename "${BASH_SOURCE[0]}") DIST-ARCH ARTIFACTORY_URL
DIST: The distribution.
ARCH: The architecture.
ARTIFACTORY_URL must contain repo path for package, including hostname.
Environment Variables
ARTIFACTORY_TOKEN: must contain an auth token. [required]
EOF
exit 1
}
set -e
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
COMPONENT_NAME="nvidia-container-toolkit"
if [[ $# -ne 1 ]]; then
assert_usage "$@"
fi
source "${SCRIPTS_DIR}"/utils.sh
DISTARCH=$1
DIST=${DISTARCH%-*}
ARCH=${DISTARCH#*-}
ARTIFACTORY_URL=$2
if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
echo "ERROR: Distro and Architecture must be specified." >&2
assert_usage "$@"
fi
# TODO: accept ARTIFACTS_DIR as a command-line argument
: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
if [[ ! -d "${ARTIFACTS_DIR}" ]]; then
echo "ERROR: ARTIFACTS_DIR does not exist." >&2
assert_usage "$@"
fi
if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then
echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2
assert_usage "$@"
fi
# TODO: accept KITMACKER_DIR as a command-line argument
: "${KITMAKER_DIR="${PROJECT_ROOT}/artifacts/kitmaker"}"
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
# Returns the key=value property if the value isn't empty
# Prepends with ";" if needed
set_prop_value() {
local key=$1
local value=$2
if [ -n "${value}" ]; then
if [ -z "${PROPS}" ]; then
echo "${key}=${value}"
else
echo ";${key}=${value}"
fi
fi
}
process_props() {
local dist=$1
local arch=$2
PROPS+=$(set_prop_value "component_name" "${COMPONENT_NAME}")
PROPS+=$(set_prop_value "version" "${VERSION}")
PROPS+=$(set_prop_value "os" "${dist}")
PROPS+=$(set_prop_value "arch" "${arch}")
PROPS+=$(set_prop_value "platform" "${dist}-${arch}")
# TODO: Use `git describe` to get this information if it's not available.
PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}")
PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}")
# Gitlab variables to expose
for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do
if [ -n "${!var}" ]; then
PROPS+=$(set_prop_value "${var}" "${!var}")
fi
done
}
## NOT USED:
## can substitute this function place of upload_file to modify properties of
## existing file instead of uploading files.
# Sets the properties on a path
# Relies on global variables: ARTIFACTORY_TOKEN, ARTIFACTORY_URL
set_props() {
local dist="$1"
local arch="$2"
local kitmakerfilename="$3"
# extract the Artifactory hostname
artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/${kitmakerfilename}"
local PROPS
process_props "${DIST}" "${ARCH}"
echo "Setting ${image_path} with properties: ${PROPS}"
if ! ${CURL} -fs -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
-X PUT \
"https://${artifactory_host}/artifactory/api/storage/${image_path}?properties=${PROPS}&recursive=0" ; then
echo "ERROR: set props failed: ${image_path}"
exit 1
fi
}
# Uploads file to ARTIFACTS_DIR/<os>/<arch>/<filename>
# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_URL
upload_file() {
local dist=$1
local arch=$2
local file=$3
# extract the Artifactory hostname
artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
# get base part of the ARTIFACTORY_URL without hostname
local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}"
local PROPS
process_props "${dist}" "${arch}"
if [ ! -r "${file}" ]; then
echo "ERROR: File not found or not readable: ${file}"
exit 1
fi
# Collect sum
SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }')
echo "Uploading ${image_path} from ${file}"
if ! ${CURL} -f \
-H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
-H "X-Checksum-Sha1: ${SHA1_SUM}" \
${file:+-T ${file}} -X PUT \
"https://${artifactory_host}/artifactory/${image_path};${PROPS}" ;
then
echo "ERROR: upload file failed: ${file}"
exit 1
fi
}
function push-kitmaker-artifactory() {
local dist=$1
local arch=$2
local archive=$3
upload_file "${dist}" "${arch}" "${archive}"
}
# kitmakerize-distro creates a tar.gz archive for the specified dist-arch combination.
# The archive is created at ${KITMAKER_DIR}/${name}.tar.gz (where ${name} is the third positional argument)
function kitmakerize-distro() {
local dist="$1"
local arch="$2"
local archive="$3"
local name=$(basename "${archive%%.tar.gz}")
## Copy packages into directory layout for .tar.gz
# TODO: make scratch_dir configurable
local scratch_dir="$(dirname ${archive})/.scratch/${name}"
local packages_dir="${scratch_dir}/.packages/"
mkdir -p "${packages_dir}"
# Copy the extracted files to the .packages directory so that a kitmaker file can be created.
source="${ARTIFACTS_DIR}/packages/${dist}/${arch}"
cp -r "${source}/"* "${packages_dir}/"
## Tar up the directory structure created above
tar zcvf "${archive}" -C "${scratch_dir}/.." "${name}"
echo "Created: ${archive}"
ls -l "${archive}"
echo "With contents:"
tar -tzvf "${archive}"
echo ""
# Clean up the scratch directories:
rm -f "${scratch_dir}/.packages/"*
rmdir "${scratch_dir}/.packages"
rmdir "${scratch_dir}"
}
kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}"
kitmaker_archive="${KITMAKER_DIR}/${kitmaker_name}.tar.gz"
kitmakerize-distro "${DIST}" "${ARCH}" "${kitmaker_archive}"
push-kitmaker-artifactory "${DIST}" "${ARCH}" "${kitmaker_archive}"

189
scripts/release-packages-artifactory.sh Executable file
View File

@ -0,0 +1,189 @@
#!/usr/bin/env bash
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# Dependencies:
# regctl
#
function assert_usage() {
cat >&2 << EOF
Incorrect arguments: $*
$(basename "${BASH_SOURCE[0]}") DIST-ARCH
DIST: The distribution.
ARCH: The architecture.
Environment Variables
ARTIFACTORY_TOKEN: must contain an auth token. [required]
LIB_TAG: optional package tag.
CI_COMMIT_REF_NAME: provided by CI/CD system.
CI_COMMIT_SHA: provided by CI/CD system.
EOF
exit 1
}
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
source "${SCRIPTS_DIR}"/utils.sh
if [[ $# -ne 1 ]]; then
assert_usage "$@"
fi
DISTARCH=$1
ARTIFACTORY_PATH=$2
DIST=${DISTARCH%-*}
ARCH=${DISTARCH#*-}
CURL=${CURL:-curl}
if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
echo "ERROR: Distro and Architecture must be specified." >&2
assert_usage "$@"
fi
if [[ -z "${ARTIFACTORY_PATH}" ]]; then
echo "ERROR: Package repo must be specified." >&2
assert_usage "$@"
fi
if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then
echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2
assert_usage "$@"
fi
# TODO: accept PACKAGES_DIR as a command-line argument
: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
: "${PACKAGES_DIR="${ARTIFACTS_DIR}/packages"}"
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
# Returns the key=value property if the value isn't empty
# Prepends with ";" if needed
set_prop_value() {
local key=$1
local value=$2
if [ -n "${value}" ]; then
if [ -z "${PROPS}" ]; then
echo "${key}=${value}"
else
echo ";${key}=${value}"
fi
fi
}
process_props() {
local dist=$1
local arch=$2
local file=$3
local component_name="${file%%.*}"
component_name="${component_name%-*}"
local pkg_type="$(package_type $dist)"
## Component owner is free to define these
# PROPS+=$(set_prop_value "version" "${VERSION}")
# PROPS+=$(set_prop_value "lws_version" "${LWS_VER}")
# PROPS+=$(set_prop_value "platform" "${DISTARCH}")
# TODO: Use `git describe` to get this information if it's not available.
PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}")
PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}")
# PROPS+=$(set_prop_value "category" "utils")
# PROPS+=$(set_prop_value "platform" "${DISTARCH}")
# Gitlab variables to expose
for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do
if [ -n "${!var}" ]; then
PROPS+=$(set_prop_value "${var}" "${!var}")
fi
done
# We also set the package-specific properties to allow this to be used for other artifactory repositories
PROPS+=$(set_prop_value "${pkg_type}.distribution" "${dist}")
PROPS+=$(set_prop_value "${pkg_type}.architecture" "${arch}")
PROPS+=$(set_prop_value "${pkg_type}.component" "${component_name}")
}
# Uploads file ARTIFACTORY_PATH
# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_PATH
upload_file() {
local dist=$1
local arch=$2
local file=$3
# TODO: These should be set by envvars
local artifactory_host="urm.nvidia.com"
local artifactory_repo="$(get_artifactory_repository $dist)"
if [ ! -r "${file}" ]; then
echo "ERROR: File not found or not readable: ${file}"
exit 1
fi
local PROPS
process_props "${dist}" "${arch}" "${file}"
# Collect sum
SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }')
url="https://${artifactory_host}/artifactory/${artifactory_repo}/${dist}/${arch}/$(basename "${file}")"
# NOTE: The URL to set the properties through the API is:
# "https://${artifactory_host}/artifactory/api/storage/${artifactory_repo}/${dist}/${arch}/$(basename ${file})"
echo "Uploading ${file} to ${url}"
if ! ${CURL} -f \
-H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
-H "X-Checksum-Sha1: ${SHA1_SUM}" \
${file:+-T ${file}} -X PUT \
"${url};${PROPS}" ;
then
echo "ERROR: upload file failed: ${file}"
exit 1
fi
}
function push-artifactory() {
local dist="$1"
local arch="$2"
source="${ARTIFACTS_DIR}/packages/${dist}/${arch}"
find "${source}" -maxdepth 1 | while read -r f ; do
upload_file "$dist" "$arch" "$f"
done
}
# TODO: use this to adapt as a general purpose command-line tool
# case "${COMMAND}" in
# set)
# set_props
# ;;
# upload)
# if [ -z "${UPLOAD_FILE}" ]; then
# echo "ERROR: Upload package filename must be set using -f"
# usage
# fi
#
# upload_file
# ;;
# *)
# echo "ERROR: Invalid command ${COMMAND}"
# usage
# ;;
# esac
push-artifactory "${DIST}" "${ARCH}"

36
scripts/utils.sh Normal file
View File

@ -0,0 +1,36 @@
# package_type returns the packaging type (deb or rpm) for the specfied distribution.
# An error is returned if the ditribution is unsupported.
function package_type() {
local pkg_type
case ${1} in
amazonlinux*) pkg_type=rpm
;;
centos*) pkg_type=rpm
;;
debian*) pkg_type=deb
;;
fedora*) pkg_type=rpm
;;
opensuse-leap*) pkg_type=rpm
;;
ubuntu*) pkg_type=deb
;;
*) exit 1
;;
esac
echo "${pkg_type}"
}
function get_artifactory_repository() {
local pkg_type=$(package_type $1)
case ${pkg_type} in
deb) echo "sw-gpu-cloudnative-debian-local"
;;
rpm) echo "sw-gpu-cloudnative-rpm-local"
;;
*) echo "sw-gpu-cloudnative-generic-local"
;;
esac
}