mirror of
https://github.com/NVIDIA/nvidia-container-toolkit
synced 2024-11-22 08:18:32 +00:00
Merge branch 'cnt-2210' into 'main'
[ci] push package releases to artifactory See merge request nvidia/container-toolkit/container-toolkit!231
This commit is contained in:
Jon Mayo
commit
28e882f26f
@ -1,4 +1,4 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
@ -47,6 +47,7 @@ stages:
|
||||
- !reference [.main-or-manual, rules]
|
||||
variables:
|
||||
DIST: amazonlinux2
|
||||
PACKAGE_REPO_TYPE: rpm
|
||||
|
||||
.dist-centos7:
|
||||
rules:
|
||||
@ -54,35 +55,41 @@ stages:
|
||||
variables:
|
||||
DIST: centos7
|
||||
CVE_UPDATES: "cyrus-sasl-lib"
|
||||
PACKAGE_REPO_TYPE: rpm
|
||||
|
||||
.dist-centos8:
|
||||
variables:
|
||||
DIST: centos8
|
||||
CVE_UPDATES: "cyrus-sasl-lib"
|
||||
PACKAGE_REPO_TYPE: rpm
|
||||
|
||||
.dist-debian10:
|
||||
rules:
|
||||
- !reference [.main-or-manual, rules]
|
||||
variables:
|
||||
DIST: debian10
|
||||
PACKAGE_REPO_TYPE: debian
|
||||
|
||||
.dist-debian9:
|
||||
rules:
|
||||
- !reference [.main-or-manual, rules]
|
||||
variables:
|
||||
DIST: debian9
|
||||
PACKAGE_REPO_TYPE: debian
|
||||
|
||||
.dist-fedora35:
|
||||
rules:
|
||||
- !reference [.main-or-manual, rules]
|
||||
variables:
|
||||
DIST: fedora35
|
||||
PACKAGE_REPO_TYPE: rpm
|
||||
|
||||
.dist-opensuse-leap15.1:
|
||||
rules:
|
||||
- !reference [.main-or-manual, rules]
|
||||
variables:
|
||||
DIST: opensuse-leap15.1
|
||||
PACKAGE_REPO_TYPE: rpm
|
||||
|
||||
.dist-ubi8:
|
||||
rules:
|
||||
@ -90,17 +97,20 @@ stages:
|
||||
variables:
|
||||
DIST: ubi8
|
||||
CVE_UPDATES: "cyrus-sasl-lib"
|
||||
PACKAGE_REPO_TYPE: rpm
|
||||
|
||||
.dist-ubuntu16.04:
|
||||
rules:
|
||||
- !reference [.main-or-manual, rules]
|
||||
variables:
|
||||
DIST: ubuntu16.04
|
||||
PACKAGE_REPO_TYPE: debian
|
||||
|
||||
.dist-ubuntu18.04:
|
||||
variables:
|
||||
DIST: ubuntu18.04
|
||||
CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
|
||||
PACKAGE_REPO_TYPE: debian
|
||||
|
||||
.dist-ubuntu20.04:
|
||||
rules:
|
||||
@ -108,6 +118,7 @@ stages:
|
||||
variables:
|
||||
DIST: ubuntu20.04
|
||||
CVE_UPDATES: "libsasl2-2 libsasl2-modules-db"
|
||||
PACKAGE_REPO_TYPE: debian
|
||||
|
||||
.dist-packaging:
|
||||
variables:
|
||||
|
||||
@ -1,4 +1,4 @@
|
||||
# Copyright (c) 2019-2021, NVIDIA CORPORATION. All rights reserved.
|
||||
# Copyright (c) 2019-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
||||
127
.nvidia-ci.yml
127
.nvidia-ci.yml
@ -1,4 +1,4 @@
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
@ -35,6 +35,7 @@ variables:
|
||||
# Define the public staging registry
|
||||
STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
|
||||
STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
|
||||
ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
|
||||
|
||||
.image-pull:
|
||||
stage: image-build
|
||||
@ -92,6 +93,109 @@ image-packaging:
|
||||
- .dist-packaging
|
||||
- .image-pull
|
||||
|
||||
# Define the package release targets
|
||||
release:packages:amazonlinux2-aarch64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-amazonlinux2
|
||||
- .arch-aarch64
|
||||
|
||||
release:packages:amazonlinux2-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-amazonlinux2
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:centos7-ppc64le:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos7
|
||||
- .arch-ppc64le
|
||||
|
||||
release:packages:centos7-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos7
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:centos8-aarch64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos8
|
||||
- .arch-aarch64
|
||||
|
||||
release:packages:centos8-ppc64le:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos8
|
||||
- .arch-ppc64le
|
||||
|
||||
release:packages:centos8-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos8
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:debian10-amd64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-debian10
|
||||
- .arch-amd64
|
||||
|
||||
release:packages:debian9-amd64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-debian9
|
||||
- .arch-amd64
|
||||
|
||||
release:packages:fedora35-aarch64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-fedora35
|
||||
- .arch-aarch64
|
||||
|
||||
release:packages:fedora35-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-fedora35
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:opensuse-leap15.1-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-opensuse-leap15.1
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:ubuntu16.04-amd64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu16.04
|
||||
- .arch-amd64
|
||||
|
||||
release:packages:ubuntu16.04-ppc64le:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu16.04
|
||||
- .arch-ppc64le
|
||||
|
||||
release:packages:ubuntu18.04-amd64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu18.04
|
||||
- .arch-amd64
|
||||
|
||||
release:packages:ubuntu18.04-arm64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu18.04
|
||||
- .arch-arm64
|
||||
|
||||
release:packages:ubuntu18.04-ppc64le:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu18.04
|
||||
- .arch-ppc64le
|
||||
|
||||
# We skip the integration tests for the internal CI:
|
||||
.integration:
|
||||
stage: test
|
||||
@ -202,6 +306,27 @@ scan-ubi8-arm64:
|
||||
OUT_REGISTRY: "${NGC_REGISTRY}"
|
||||
OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
|
||||
|
||||
.release:packages:
|
||||
stage: release
|
||||
needs:
|
||||
- image-packaging
|
||||
variables:
|
||||
PACKAGE_REGISTRY: "${CI_REGISTRY}"
|
||||
PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
|
||||
PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
|
||||
PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
|
||||
PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
|
||||
PACKAGE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-${PACKAGE_REPO_TYPE}-local"
|
||||
KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/kitmaker"
|
||||
script:
|
||||
- !reference [.regctl-setup, before_script]
|
||||
- apk add --no-cache bash
|
||||
- regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
|
||||
- ./scripts/extract-packages.sh "${PACKAGE_REGISTRY}/${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}"
|
||||
# TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
|
||||
- ./scripts/kitmakerize-packages.sh" "${DIST}-${ARCH}"
|
||||
- ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}"
|
||||
|
||||
release:staging-ubuntu18.04:
|
||||
extends:
|
||||
- .release:staging
|
||||
|
||||
@ -25,5 +25,7 @@ ARG ARTIFACTS_ROOT
|
||||
COPY ${ARTIFACTS_ROOT} /artifacts/packages/
|
||||
|
||||
WORKDIR /artifacts/packages
|
||||
# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
|
||||
RUN find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' > /artifacts/manifest.txt
|
||||
|
||||
RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
|
||||
|
||||
@ -21,14 +21,14 @@
|
||||
|
||||
function assert_usage() {
|
||||
echo "Missing argument $1"
|
||||
echo "$(basename ${BASH_SOURCE[0]}) TARGET"
|
||||
echo "$(basename "${BASH_SOURCE[0]}") TARGET"
|
||||
exit 1
|
||||
}
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
|
||||
PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
|
||||
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}"/.. && pwd )"
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
assert_usage "TARGET"
|
||||
@ -36,31 +36,31 @@ fi
|
||||
|
||||
TARGET=$1
|
||||
|
||||
: ${DIST_DIR:=${PROJECT_ROOT}/dist}
|
||||
: "${DIST_DIR:=${PROJECT_ROOT}/dist}"
|
||||
export DIST_DIR
|
||||
|
||||
echo "Building ${TARGET} for all packages to ${DIST_DIR}"
|
||||
|
||||
: ${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}
|
||||
: ${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}
|
||||
: ${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}
|
||||
: ${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}
|
||||
: "${LIBNVIDIA_CONTAINER_ROOT:=${PROJECT_ROOT}/third_party/libnvidia-container}"
|
||||
: "${NVIDIA_CONTAINER_TOOLKIT_ROOT:=${PROJECT_ROOT}}"
|
||||
: "${NVIDIA_CONTAINER_RUNTIME_ROOT:=${PROJECT_ROOT}/third_party/nvidia-container-runtime}"
|
||||
: "${NVIDIA_DOCKER_ROOT:=${PROJECT_ROOT}/third_party/nvidia-docker}"
|
||||
|
||||
|
||||
${SCRIPTS_DIR}/get-component-versions.sh
|
||||
"${SCRIPTS_DIR}/get-component-versions.sh"
|
||||
|
||||
# Build libnvidia-container
|
||||
make -C ${LIBNVIDIA_CONTAINER_ROOT} -f mk/docker.mk ${TARGET}
|
||||
make -C "${LIBNVIDIA_CONTAINER_ROOT}" -f mk/docker.mk "${TARGET}"
|
||||
|
||||
if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_VERSION} || -z ${LIBNVIDIA_CONTAINER_VERSION} ]]; then
|
||||
if [[ -z "${NVIDIA_CONTAINER_TOOLKIT_VERSION}" || -z "${LIBNVIDIA_CONTAINER_VERSION}" ]]; then
|
||||
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
|
||||
fi
|
||||
|
||||
# Build nvidia-container-toolkit
|
||||
make -C ${NVIDIA_CONTAINER_TOOLKIT_ROOT} \
|
||||
make -C "${NVIDIA_CONTAINER_TOOLKIT_ROOT}" \
|
||||
LIBNVIDIA_CONTAINER_VERSION="${LIBNVIDIA_CONTAINER_VERSION}" \
|
||||
LIBNVIDIA_CONTAINER_TAG="${LIBNVIDIA_CONTAINER_TAG}" \
|
||||
${TARGET}
|
||||
"${TARGET}"
|
||||
|
||||
if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then
|
||||
# We set the TOOLKIT_VERSION, TOOLKIT_TAG for the nvidia-container-runtime and nvidia-docker targets
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved.
|
||||
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
@ -22,7 +22,6 @@
|
||||
set -e
|
||||
|
||||
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
|
||||
PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
|
||||
|
||||
# This list represents the distribution-architecture pairs that are actually published
|
||||
# to the relevant repositories. This targets forwarded to the build-all-components script
|
||||
@ -54,7 +53,7 @@ else
|
||||
fi
|
||||
|
||||
echo "Updating components"
|
||||
${SCRIPTS_DIR}/update-components.sh
|
||||
"${SCRIPTS_DIR}/update-components.sh"
|
||||
if [[ -n $(git status -s third_party) && ${ALLOW_LOCAL_COMPONENT_CHANGES} != "true" ]]; then
|
||||
echo "ERROR: Building with local component changes."
|
||||
echo "Commit pending changes or rerun with ALLOW_LOCAL_COMPONENT_CHANGES='true'"
|
||||
@ -66,7 +65,7 @@ eval $(${SCRIPTS_DIR}/get-component-versions.sh)
|
||||
|
||||
if [[ -n ${NVIDIA_CONTAINER_TOOLKIT_TAG} ]]; then
|
||||
echo "Allowing mismatched versions for release candidate "
|
||||
: ${ALLOW_VERSION_MISMATCH:=true}
|
||||
: "${ALLOW_VERSION_MISMATCH:=true}"
|
||||
fi
|
||||
|
||||
if [[ "${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}" != "${LIBNVIDIA_CONTAINER_PACKAGE_VERSION}" ]]; then
|
||||
@ -86,6 +85,6 @@ export LIBNVIDIA_CONTAINER_TAG
|
||||
export NVIDIA_CONTAINER_RUNTIME_VERSION
|
||||
export NVIDIA_DOCKER_VERSION
|
||||
|
||||
for target in ${targets[@]}; do
|
||||
${SCRIPTS_DIR}/build-all-components.sh ${target}
|
||||
for target in "${targets[@]}"; do
|
||||
"${SCRIPTS_DIR}/build-all-components.sh" "${target}"
|
||||
done
|
||||
|
||||
112
scripts/extract-packages.sh
Executable file
112
scripts/extract-packages.sh
Executable file
@ -0,0 +1,112 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
function assert_usage() {
|
||||
echo "Incorrect arguments: $*" >&2
|
||||
echo "$(basename "${BASH_SOURCE[0]}") PACKAGE_IMAGE_NAME:PACKAGE_IMAGE_TAG DIST-ARCH" >&2
|
||||
echo -e "\\tPACKAGE_IMAGE: container image holding packages [e.g. registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit]" >&2
|
||||
echo -e "\\tPACKAGE_TAG: tag for container image holding packages. [e.g. 1a2b3c4-packaging]" >&2
|
||||
echo -e "\\tDIST: The distribution." >&2
|
||||
echo -e "\\tARCH: The architecture." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
|
||||
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
|
||||
|
||||
if [[ $# -ne 2 ]]; then
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
PACKAGE_IMAGE=$1
|
||||
DISTARCH=$2
|
||||
DIST=${DISTARCH%-*}
|
||||
ARCH=${DISTARCH#*-}
|
||||
|
||||
if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
|
||||
echo "ERROR: Distro and Architecture must be specified." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
# TODO: accept ARTIFACTS_DIR as a command-line argument
|
||||
: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
|
||||
|
||||
# For release-candidates we skip certain packages.
|
||||
# For example, we don't release release candidates of nvidia-container-runtime and nvidia-docker2
|
||||
# since these only bump the nvidia-container-toolkit dependency.
|
||||
function skip-for-release-candidate() {
|
||||
if [[ "${VERSION/rc./}" == "${VERSION}" ]]; then
|
||||
return 1
|
||||
fi
|
||||
|
||||
local package_name=$1
|
||||
if [[ "${package_name/"nvidia-docker2"/}" != "${package_name}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
if [[ "${package_name/"nvidia-container-runtime"/}" != "${package_name}" ]]; then
|
||||
return 0
|
||||
fi
|
||||
return 1
|
||||
}
|
||||
|
||||
# extract-file copies a file from a specified image.
|
||||
# If regctl is available this is used, otherwise a docker container is run and the file is copied from
|
||||
# there.
|
||||
function copy-file() {
|
||||
local image=$1
|
||||
local path_in_image=$2
|
||||
local path_on_host=$3
|
||||
if command -v regctl; then
|
||||
regctl image get-file "${image}" "${path_in_image}" "${path_on_host}"
|
||||
else
|
||||
# Note this will only work for destinations where the `path_on_host` is in `pwd`
|
||||
docker run --rm \
|
||||
-v "$(pwd):$(pwd)" \
|
||||
-w "$(pwd)" \
|
||||
-u "$(id -u):$(id -g)" \
|
||||
--entrypoint="bash" \
|
||||
"${image}" \
|
||||
-c "cp ${path_in_image} ${path_on_host}"
|
||||
fi
|
||||
}
|
||||
|
||||
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
|
||||
|
||||
# extract-all extracts all package for the specified dist-arch combination from the package image.
|
||||
# The manifest.txt file in the image is used to detemine the applicable files for the combination.
|
||||
# Files are extracted to ${ARTIFACTS_DIR}/artifacts/packages/${dist}/${arch}
|
||||
function extract-all() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
|
||||
mkdir -p "${ARTIFACTS_DIR}"
|
||||
copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt"
|
||||
|
||||
# Extract every file for the specified dist-arch combiniation in MANIFEST.txt
|
||||
grep "/${dist}/${arch}/" "${ARTIFACTS_DIR}/manifest.txt" | while read -r f ; do
|
||||
package_name="$(basename "$f")"
|
||||
# For release-candidates, we skip certain packages
|
||||
if skip-for-release-candidate "${package_name}"; then
|
||||
echo "Skipping $f for release-candidate ${VERSION}"
|
||||
continue
|
||||
fi
|
||||
target="${ARTIFACTS_DIR}/packages/${dist}/${arch}/${package_name}"
|
||||
mkdir -p "$(dirname "$target")"
|
||||
copy-file "${PACKAGE_IMAGE}" "${f}" "${target}"
|
||||
done
|
||||
}
|
||||
|
||||
extract-all "${DIST}" "${ARCH}"
|
||||
@ -9,7 +9,7 @@ set -x -e
|
||||
|
||||
function deb-sign {
|
||||
local last_found
|
||||
for r in ${*}; do
|
||||
for r in "$@"; do
|
||||
if [ -f "./${r}" ]; then
|
||||
last_found=${r}
|
||||
fi
|
||||
@ -27,12 +27,12 @@ function deb-sign {
|
||||
--no-emit-version \
|
||||
--no-comments \
|
||||
--personal-digest-preferences sha512 \
|
||||
--local-user ${GPG_LOCAL_USER} \
|
||||
--local-user "${GPG_LOCAL_USER}" \
|
||||
> InRelease
|
||||
}
|
||||
|
||||
function rpm-sign {
|
||||
for r in ${*}; do
|
||||
for r in "$@"; do
|
||||
if [ -f "./${r}" ]; then
|
||||
rpmsign --addsign --key-id A04EA552 --digest-algo=sha512 "${r}"
|
||||
fi
|
||||
@ -42,7 +42,7 @@ function rpm-sign {
|
||||
--armor \
|
||||
--no-emit-version \
|
||||
--no-comments --personal-digest-preferences sha512 \
|
||||
--local-user ${GPG_LOCAL_USER} \
|
||||
--local-user "${GPG_LOCAL_USER}" \
|
||||
repodata/repomd.xml
|
||||
}
|
||||
|
||||
@ -83,15 +83,15 @@ function sign() {
|
||||
return
|
||||
fi
|
||||
|
||||
cd ${dst}
|
||||
cd "${dst}"
|
||||
if [[ -f "/etc/debian_version" ]]; then
|
||||
[[ ${pkg_type} == "deb" ]] && deb-sign ${ALL_DEBS}
|
||||
[[ "${pkg_type}" == "deb" ]] && deb-sign ${ALL_DEBS}
|
||||
else
|
||||
[[ ${pkg_type} == "rpm" ]] && rpm-sign ${ALL_RPMS}
|
||||
[[ "${pkg_type}" == "rpm" ]] && rpm-sign ${ALL_RPMS}
|
||||
fi
|
||||
cd -
|
||||
}
|
||||
|
||||
for target in ${TARGETS[@]}; do
|
||||
sign ${target} $(pwd)
|
||||
for target in "${TARGETS[@]}"; do
|
||||
sign "${target}" "$(pwd)"
|
||||
done
|
||||
|
||||
211
scripts/release-kitmaker-artifactory.sh
Executable file
211
scripts/release-kitmaker-artifactory.sh
Executable file
@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
function assert_usage() {
|
||||
cat >&2 << EOF
|
||||
Incorrect arguments: $*
|
||||
$(basename "${BASH_SOURCE[0]}") DIST-ARCH ARTIFACTORY_URL
|
||||
DIST: The distribution.
|
||||
ARCH: The architecture.
|
||||
ARTIFACTORY_URL must contain repo path for package, including hostname.
|
||||
|
||||
Environment Variables
|
||||
ARTIFACTORY_TOKEN: must contain an auth token. [required]
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
|
||||
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
|
||||
COMPONENT_NAME="nvidia-container-toolkit"
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
source "${SCRIPTS_DIR}"/utils.sh
|
||||
|
||||
DISTARCH=$1
|
||||
DIST=${DISTARCH%-*}
|
||||
ARCH=${DISTARCH#*-}
|
||||
ARTIFACTORY_URL=$2
|
||||
|
||||
if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
|
||||
echo "ERROR: Distro and Architecture must be specified." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
# TODO: accept ARTIFACTS_DIR as a command-line argument
|
||||
: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
|
||||
|
||||
if [[ ! -d "${ARTIFACTS_DIR}" ]]; then
|
||||
echo "ERROR: ARTIFACTS_DIR does not exist." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then
|
||||
echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
# TODO: accept KITMACKER_DIR as a command-line argument
|
||||
: "${KITMAKER_DIR="${PROJECT_ROOT}/artifacts/kitmaker"}"
|
||||
|
||||
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
|
||||
|
||||
# Returns the key=value property if the value isn't empty
|
||||
# Prepends with ";" if needed
|
||||
set_prop_value() {
|
||||
local key=$1
|
||||
local value=$2
|
||||
if [ -n "${value}" ]; then
|
||||
if [ -z "${PROPS}" ]; then
|
||||
echo "${key}=${value}"
|
||||
else
|
||||
echo ";${key}=${value}"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
process_props() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
|
||||
PROPS+=$(set_prop_value "component_name" "${COMPONENT_NAME}")
|
||||
PROPS+=$(set_prop_value "version" "${VERSION}")
|
||||
PROPS+=$(set_prop_value "os" "${dist}")
|
||||
PROPS+=$(set_prop_value "arch" "${arch}")
|
||||
PROPS+=$(set_prop_value "platform" "${dist}-${arch}")
|
||||
# TODO: Use `git describe` to get this information if it's not available.
|
||||
PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}")
|
||||
PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}")
|
||||
|
||||
# Gitlab variables to expose
|
||||
for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do
|
||||
if [ -n "${!var}" ]; then
|
||||
PROPS+=$(set_prop_value "${var}" "${!var}")
|
||||
fi
|
||||
done
|
||||
}
|
||||
|
||||
## NOT USED:
|
||||
## can substitute this function place of upload_file to modify properties of
|
||||
## existing file instead of uploading files.
|
||||
# Sets the properties on a path
|
||||
# Relies on global variables: ARTIFACTORY_TOKEN, ARTIFACTORY_URL
|
||||
set_props() {
|
||||
local dist="$1"
|
||||
local arch="$2"
|
||||
local kitmakerfilename="$3"
|
||||
|
||||
# extract the Artifactory hostname
|
||||
artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
|
||||
local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/${kitmakerfilename}"
|
||||
|
||||
local PROPS
|
||||
process_props "${DIST}" "${ARCH}"
|
||||
|
||||
echo "Setting ${image_path} with properties: ${PROPS}"
|
||||
if ! ${CURL} -fs -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
|
||||
-X PUT \
|
||||
"https://${artifactory_host}/artifactory/api/storage/${image_path}?properties=${PROPS}&recursive=0" ; then
|
||||
echo "ERROR: set props failed: ${image_path}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Uploads file to ARTIFACTS_DIR/<os>/<arch>/<filename>
|
||||
# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_URL
|
||||
upload_file() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
local file=$3
|
||||
|
||||
# extract the Artifactory hostname
|
||||
artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
|
||||
# get base part of the ARTIFACTORY_URL without hostname
|
||||
local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}"
|
||||
|
||||
local PROPS
|
||||
process_props "${dist}" "${arch}"
|
||||
|
||||
if [ ! -r "${file}" ]; then
|
||||
echo "ERROR: File not found or not readable: ${file}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Collect sum
|
||||
SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }')
|
||||
|
||||
echo "Uploading ${image_path} from ${file}"
|
||||
if ! ${CURL} -f \
|
||||
-H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
|
||||
-H "X-Checksum-Sha1: ${SHA1_SUM}" \
|
||||
${file:+-T ${file}} -X PUT \
|
||||
"https://${artifactory_host}/artifactory/${image_path};${PROPS}" ;
|
||||
then
|
||||
echo "ERROR: upload file failed: ${file}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function push-kitmaker-artifactory() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
local archive=$3
|
||||
|
||||
upload_file "${dist}" "${arch}" "${archive}"
|
||||
}
|
||||
|
||||
# kitmakerize-distro creates a tar.gz archive for the specified dist-arch combination.
|
||||
# The archive is created at ${KITMAKER_DIR}/${name}.tar.gz (where ${name} is the third positional argument)
|
||||
function kitmakerize-distro() {
|
||||
local dist="$1"
|
||||
local arch="$2"
|
||||
local archive="$3"
|
||||
|
||||
local name=$(basename "${archive%%.tar.gz}")
|
||||
## Copy packages into directory layout for .tar.gz
|
||||
# TODO: make scratch_dir configurable
|
||||
local scratch_dir="$(dirname ${archive})/.scratch/${name}"
|
||||
local packages_dir="${scratch_dir}/.packages/"
|
||||
|
||||
mkdir -p "${packages_dir}"
|
||||
|
||||
# Copy the extracted files to the .packages directory so that a kitmaker file can be created.
|
||||
source="${ARTIFACTS_DIR}/packages/${dist}/${arch}"
|
||||
cp -r "${source}/"* "${packages_dir}/"
|
||||
|
||||
## Tar up the directory structure created above
|
||||
tar zcvf "${archive}" -C "${scratch_dir}/.." "${name}"
|
||||
echo "Created: ${archive}"
|
||||
ls -l "${archive}"
|
||||
echo "With contents:"
|
||||
tar -tzvf "${archive}"
|
||||
echo ""
|
||||
|
||||
# Clean up the scratch directories:
|
||||
rm -f "${scratch_dir}/.packages/"*
|
||||
rmdir "${scratch_dir}/.packages"
|
||||
rmdir "${scratch_dir}"
|
||||
}
|
||||
|
||||
kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION}"
|
||||
kitmaker_archive="${KITMAKER_DIR}/${kitmaker_name}.tar.gz"
|
||||
kitmakerize-distro "${DIST}" "${ARCH}" "${kitmaker_archive}"
|
||||
push-kitmaker-artifactory "${DIST}" "${ARCH}" "${kitmaker_archive}"
|
||||
189
scripts/release-packages-artifactory.sh
Executable file
189
scripts/release-packages-artifactory.sh
Executable file
@ -0,0 +1,189 @@
|
||||
#!/usr/bin/env bash
|
||||
|
||||
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Dependencies:
|
||||
# regctl
|
||||
#
|
||||
|
||||
function assert_usage() {
|
||||
cat >&2 << EOF
|
||||
Incorrect arguments: $*
|
||||
$(basename "${BASH_SOURCE[0]}") DIST-ARCH
|
||||
DIST: The distribution.
|
||||
ARCH: The architecture.
|
||||
|
||||
Environment Variables
|
||||
ARTIFACTORY_TOKEN: must contain an auth token. [required]
|
||||
LIB_TAG: optional package tag.
|
||||
CI_COMMIT_REF_NAME: provided by CI/CD system.
|
||||
CI_COMMIT_SHA: provided by CI/CD system.
|
||||
EOF
|
||||
exit 1
|
||||
}
|
||||
|
||||
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
|
||||
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
|
||||
|
||||
source "${SCRIPTS_DIR}"/utils.sh
|
||||
|
||||
if [[ $# -ne 1 ]]; then
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
DISTARCH=$1
|
||||
ARTIFACTORY_PATH=$2
|
||||
DIST=${DISTARCH%-*}
|
||||
ARCH=${DISTARCH#*-}
|
||||
|
||||
CURL=${CURL:-curl}
|
||||
|
||||
if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
|
||||
echo "ERROR: Distro and Architecture must be specified." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
if [[ -z "${ARTIFACTORY_PATH}" ]]; then
|
||||
echo "ERROR: Package repo must be specified." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then
|
||||
echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
# TODO: accept PACKAGES_DIR as a command-line argument
|
||||
: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
|
||||
: "${PACKAGES_DIR="${ARTIFACTS_DIR}/packages"}"
|
||||
|
||||
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
|
||||
|
||||
# Returns the key=value property if the value isn't empty
|
||||
# Prepends with ";" if needed
|
||||
set_prop_value() {
|
||||
local key=$1
|
||||
local value=$2
|
||||
if [ -n "${value}" ]; then
|
||||
if [ -z "${PROPS}" ]; then
|
||||
echo "${key}=${value}"
|
||||
else
|
||||
echo ";${key}=${value}"
|
||||
fi
|
||||
fi
|
||||
}
|
||||
|
||||
process_props() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
local file=$3
|
||||
local component_name="${file%%.*}"
|
||||
component_name="${component_name%-*}"
|
||||
local pkg_type="$(package_type $dist)"
|
||||
|
||||
## Component owner is free to define these
|
||||
# PROPS+=$(set_prop_value "version" "${VERSION}")
|
||||
# PROPS+=$(set_prop_value "lws_version" "${LWS_VER}")
|
||||
# PROPS+=$(set_prop_value "platform" "${DISTARCH}")
|
||||
|
||||
# TODO: Use `git describe` to get this information if it's not available.
|
||||
PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}")
|
||||
PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}")
|
||||
|
||||
# PROPS+=$(set_prop_value "category" "utils")
|
||||
# PROPS+=$(set_prop_value "platform" "${DISTARCH}")
|
||||
|
||||
# Gitlab variables to expose
|
||||
for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do
|
||||
if [ -n "${!var}" ]; then
|
||||
PROPS+=$(set_prop_value "${var}" "${!var}")
|
||||
fi
|
||||
done
|
||||
|
||||
# We also set the package-specific properties to allow this to be used for other artifactory repositories
|
||||
PROPS+=$(set_prop_value "${pkg_type}.distribution" "${dist}")
|
||||
PROPS+=$(set_prop_value "${pkg_type}.architecture" "${arch}")
|
||||
PROPS+=$(set_prop_value "${pkg_type}.component" "${component_name}")
|
||||
}
|
||||
|
||||
# Uploads file ARTIFACTORY_PATH
|
||||
# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_PATH
|
||||
upload_file() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
local file=$3
|
||||
|
||||
# TODO: These should be set by envvars
|
||||
local artifactory_host="urm.nvidia.com"
|
||||
local artifactory_repo="$(get_artifactory_repository $dist)"
|
||||
|
||||
if [ ! -r "${file}" ]; then
|
||||
echo "ERROR: File not found or not readable: ${file}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
local PROPS
|
||||
process_props "${dist}" "${arch}" "${file}"
|
||||
|
||||
# Collect sum
|
||||
SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }')
|
||||
|
||||
url="https://${artifactory_host}/artifactory/${artifactory_repo}/${dist}/${arch}/$(basename "${file}")"
|
||||
# NOTE: The URL to set the properties through the API is:
|
||||
# "https://${artifactory_host}/artifactory/api/storage/${artifactory_repo}/${dist}/${arch}/$(basename ${file})"
|
||||
|
||||
echo "Uploading ${file} to ${url}"
|
||||
if ! ${CURL} -f \
|
||||
-H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
|
||||
-H "X-Checksum-Sha1: ${SHA1_SUM}" \
|
||||
${file:+-T ${file}} -X PUT \
|
||||
"${url};${PROPS}" ;
|
||||
then
|
||||
echo "ERROR: upload file failed: ${file}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function push-artifactory() {
|
||||
local dist="$1"
|
||||
local arch="$2"
|
||||
|
||||
source="${ARTIFACTS_DIR}/packages/${dist}/${arch}"
|
||||
|
||||
find "${source}" -maxdepth 1 | while read -r f ; do
|
||||
upload_file "$dist" "$arch" "$f"
|
||||
done
|
||||
}
|
||||
|
||||
# TODO: use this to adapt as a general purpose command-line tool
|
||||
# case "${COMMAND}" in
|
||||
# set)
|
||||
# set_props
|
||||
# ;;
|
||||
# upload)
|
||||
# if [ -z "${UPLOAD_FILE}" ]; then
|
||||
# echo "ERROR: Upload package filename must be set using -f"
|
||||
# usage
|
||||
# fi
|
||||
#
|
||||
# upload_file
|
||||
# ;;
|
||||
# *)
|
||||
# echo "ERROR: Invalid command ${COMMAND}"
|
||||
# usage
|
||||
# ;;
|
||||
# esac
|
||||
|
||||
push-artifactory "${DIST}" "${ARCH}"
|
||||
36
scripts/utils.sh
Normal file
36
scripts/utils.sh
Normal file
@ -0,0 +1,36 @@
|
||||
|
||||
# package_type returns the packaging type (deb or rpm) for the specfied distribution.
|
||||
# An error is returned if the ditribution is unsupported.
|
||||
function package_type() {
|
||||
local pkg_type
|
||||
case ${1} in
|
||||
amazonlinux*) pkg_type=rpm
|
||||
;;
|
||||
centos*) pkg_type=rpm
|
||||
;;
|
||||
debian*) pkg_type=deb
|
||||
;;
|
||||
fedora*) pkg_type=rpm
|
||||
;;
|
||||
opensuse-leap*) pkg_type=rpm
|
||||
;;
|
||||
ubuntu*) pkg_type=deb
|
||||
;;
|
||||
*) exit 1
|
||||
;;
|
||||
esac
|
||||
echo "${pkg_type}"
|
||||
}
|
||||
|
||||
function get_artifactory_repository() {
|
||||
local pkg_type=$(package_type $1)
|
||||
|
||||
case ${pkg_type} in
|
||||
deb) echo "sw-gpu-cloudnative-debian-local"
|
||||
;;
|
||||
rpm) echo "sw-gpu-cloudnative-rpm-local"
|
||||
;;
|
||||
*) echo "sw-gpu-cloudnative-generic-local"
|
||||
;;
|
||||
esac
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user