mirror of
https://github.com/NVIDIA/nvidia-container-toolkit
synced 2024-11-24 21:14:00 +00:00
Merge branch 'kitmaker-update' into 'main'
Update the process for publishing packages to kitmaker See merge request nvidia/container-toolkit/container-toolkit!271
This commit is contained in:
Evan Lezar
commit
9c5c12a1bc
115
.nvidia-ci.yml
115
.nvidia-ci.yml
@ -95,109 +95,6 @@ image-packaging:
|
||||
- .dist-packaging
|
||||
- .image-pull
|
||||
|
||||
# Define the package release targets
|
||||
release:packages:amazonlinux2-aarch64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-amazonlinux2
|
||||
- .arch-aarch64
|
||||
|
||||
release:packages:amazonlinux2-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-amazonlinux2
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:centos7-ppc64le:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos7
|
||||
- .arch-ppc64le
|
||||
|
||||
release:packages:centos7-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos7
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:centos8-aarch64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos8
|
||||
- .arch-aarch64
|
||||
|
||||
release:packages:centos8-ppc64le:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos8
|
||||
- .arch-ppc64le
|
||||
|
||||
release:packages:centos8-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-centos8
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:debian10-amd64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-debian10
|
||||
- .arch-amd64
|
||||
|
||||
release:packages:debian9-amd64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-debian9
|
||||
- .arch-amd64
|
||||
|
||||
release:packages:fedora35-aarch64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-fedora35
|
||||
- .arch-aarch64
|
||||
|
||||
release:packages:fedora35-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-fedora35
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:opensuse-leap15.1-x86_64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-opensuse-leap15.1
|
||||
- .arch-x86_64
|
||||
|
||||
release:packages:ubuntu16.04-amd64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu16.04
|
||||
- .arch-amd64
|
||||
|
||||
release:packages:ubuntu16.04-ppc64le:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu16.04
|
||||
- .arch-ppc64le
|
||||
|
||||
release:packages:ubuntu18.04-amd64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu18.04
|
||||
- .arch-amd64
|
||||
|
||||
release:packages:ubuntu18.04-arm64:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu18.04
|
||||
- .arch-arm64
|
||||
|
||||
release:packages:ubuntu18.04-ppc64le:
|
||||
extends:
|
||||
- .release:packages
|
||||
- .dist-ubuntu18.04
|
||||
- .arch-ppc64le
|
||||
|
||||
# We skip the integration tests for the internal CI:
|
||||
.integration:
|
||||
stage: test
|
||||
@ -319,15 +216,19 @@ scan-ubi8-arm64:
|
||||
PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
|
||||
PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
|
||||
PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
|
||||
PACKAGE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-${PACKAGE_REPO_TYPE}-local"
|
||||
KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
|
||||
script:
|
||||
- !reference [.regctl-setup, before_script]
|
||||
- apk add --no-cache bash
|
||||
- apk add --no-cache bash git
|
||||
- regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
|
||||
- ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}" "${DIST}-${ARCH}"
|
||||
- ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
|
||||
# TODO: ./scripts/release-packages-artifactory.sh "${DIST}-${ARCH}" "${PACKAGE_ARTIFACTORY_REPO}"
|
||||
- ./scripts/release-kitmaker-artifactory.sh "${DIST}-${ARCH}" "${KITMAKER_ARTIFACTORY_REPO}"
|
||||
- ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
|
||||
|
||||
# Define the package release targets
|
||||
release:packages:kitmaker:
|
||||
extends:
|
||||
- .release:packages
|
||||
|
||||
release:staging-ubuntu18.04:
|
||||
extends:
|
||||
|
||||
@ -15,17 +15,23 @@
|
||||
ARG BASE_DIST
|
||||
ARG CUDA_VERSION
|
||||
ARG GOLANG_VERSION=x.x.x
|
||||
ARG VERSION="N/A"
|
||||
|
||||
FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
|
||||
|
||||
ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
|
||||
ARG VERSION="N/A"
|
||||
|
||||
ARG ARTIFACTS_ROOT
|
||||
COPY ${ARTIFACTS_ROOT} /artifacts/packages/
|
||||
|
||||
WORKDIR /artifacts/packages
|
||||
|
||||
ARG GIT_BRANCH
|
||||
ARG GIT_COMMIT
|
||||
ARG SOURCE_DATE_EPOCH
|
||||
|
||||
# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
|
||||
RUN find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' > /artifacts/manifest.txt
|
||||
RUN echo "IMAGE_EPOCH=$(date '+%s')" | sed 's/^/#/g' > /artifacts/manifest.txt && \
|
||||
env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
|
||||
find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
|
||||
|
||||
RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
|
||||
|
||||
@ -94,6 +94,9 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
|
||||
--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
|
||||
--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
|
||||
--build-arg VERSION="$(VERSION)" \
|
||||
--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
|
||||
--build-arg GIT_BRANCH="$(GIT_BRANCH)" \
|
||||
--build-arg SOURCE_DATE_EPOCH="$(SOURCE_DATE_EPOCH)" \
|
||||
--build-arg CVE_UPDATES="$(CVE_UPDATES)" \
|
||||
-f $(DOCKERFILE) \
|
||||
$(CURDIR)
|
||||
|
||||
@ -16,30 +16,22 @@
|
||||
|
||||
function assert_usage() {
|
||||
echo "Incorrect arguments: $*" >&2
|
||||
echo "$(basename "${BASH_SOURCE[0]}") PACKAGE_IMAGE_NAME:PACKAGE_IMAGE_TAG DIST-ARCH" >&2
|
||||
echo "$(basename "${BASH_SOURCE[0]}") PACKAGE_IMAGE_NAME:PACKAGE_IMAGE_TAG" >&2
|
||||
echo -e "\\tPACKAGE_IMAGE: container image holding packages [e.g. registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging/container-toolkit]" >&2
|
||||
echo -e "\\tPACKAGE_TAG: tag for container image holding packages. [e.g. 1a2b3c4-packaging]" >&2
|
||||
echo -e "\\tDIST: The distribution." >&2
|
||||
echo -e "\\tARCH: The architecture." >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
|
||||
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
|
||||
|
||||
if [[ $# -ne 2 ]]; then
|
||||
if [[ $# -ne 1 ]]; then
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
PACKAGE_IMAGE=$1
|
||||
DISTARCH=$2
|
||||
DIST=${DISTARCH%-*}
|
||||
ARCH=${DISTARCH##*-}
|
||||
|
||||
if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
|
||||
echo "ERROR: Distro and Architecture must be specified." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
# TODO: accept ARTIFACTS_DIR as a command-line argument
|
||||
: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
|
||||
@ -85,30 +77,30 @@ function copy-file() {
|
||||
|
||||
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
|
||||
|
||||
# extract-all extracts all package for the specified dist-arch combination from the package image.
|
||||
# extract-all extracts all package for the specified distribution from the package image.
|
||||
# The manifest.txt file in the image is used to detemine the applicable files for the combination.
|
||||
# Files are extracted to ${ARTIFACTS_DIR}/artifacts/packages/${dist}/${arch}
|
||||
function extract-all() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
|
||||
echo "Extracting packages for ${dist}-${arch} from ${PACKAGE_IMAGE}"
|
||||
|
||||
mkdir -p "${ARTIFACTS_DIR}"
|
||||
copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt"
|
||||
echo "Extracting packages for ${dist} from ${PACKAGE_IMAGE}"
|
||||
|
||||
# Extract every file for the specified dist-arch combiniation in MANIFEST.txt
|
||||
grep "/${dist}/${arch}/" "${ARTIFACTS_DIR}/manifest.txt" | while read -r f ; do
|
||||
grep "/${dist}/" "${ARTIFACTS_DIR}/manifest.txt" | while read -r f ; do
|
||||
package_name="$(basename "$f")"
|
||||
# For release-candidates, we skip certain packages
|
||||
if skip-for-release-candidate "${package_name}"; then
|
||||
echo "Skipping $f for release-candidate ${VERSION}"
|
||||
continue
|
||||
fi
|
||||
target="${ARTIFACTS_DIR}/packages/${dist}/${arch}/${package_name}"
|
||||
target="${ARTIFACTS_DIR}/${f##/artifacts/}"
|
||||
mkdir -p "$(dirname "$target")"
|
||||
copy-file "${PACKAGE_IMAGE}" "${f}" "${target}"
|
||||
done
|
||||
}
|
||||
|
||||
extract-all "${DIST}" "${ARCH}"
|
||||
mkdir -p "${ARTIFACTS_DIR}"
|
||||
copy-file "${PACKAGE_IMAGE}" "/artifacts/manifest.txt" "${ARTIFACTS_DIR}/manifest.txt"
|
||||
|
||||
extract-all ubuntu18.04
|
||||
extract-all centos8
|
||||
|
||||
@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env bash
|
||||
#!/bin/bash
|
||||
|
||||
# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved.
|
||||
# Copyright (c) NVIDIA CORPORATION. All rights reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
@ -17,10 +17,8 @@
|
||||
function assert_usage() {
|
||||
cat >&2 << EOF
|
||||
Incorrect arguments: $*
|
||||
$(basename "${BASH_SOURCE[0]}") DIST-ARCH ARTIFACTORY_URL
|
||||
DIST: The distribution.
|
||||
ARCH: The architecture.
|
||||
ARTIFACTORY_URL must contain repo path for package, including hostname.
|
||||
$(basename "${BASH_SOURCE[0]}") KITMAKER_ARTIFACTORY_REPO
|
||||
KITMAKER_ARTIFACTORY_REPO must contain repo path for package, including hostname.
|
||||
|
||||
Environment Variables
|
||||
ARTIFACTORY_TOKEN: must contain an auth token. [required]
|
||||
@ -32,26 +30,20 @@ set -e
|
||||
|
||||
SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
|
||||
PROJECT_ROOT="$( cd "${SCRIPTS_DIR}/.." && pwd )"
|
||||
COMPONENT_NAME="nvidia-container-toolkit"
|
||||
|
||||
if [[ $# -ne 2 ]]; then
|
||||
if [[ $# -ne 1 ]]; then
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
source "${SCRIPTS_DIR}"/utils.sh
|
||||
|
||||
DISTARCH=$1
|
||||
DIST=${DISTARCH%-*}
|
||||
ARCH=${DISTARCH##*-}
|
||||
ARTIFACTORY_URL=$2
|
||||
# KITMAKER_ARTIFACTORY_REPO=https://urm.nvidia.com/artifactory/sw-gpu-cloudnative-generic-local/testing
|
||||
KITMAKER_ARTIFACTORY_REPO=$1
|
||||
|
||||
CURL=${CURL:-curl}
|
||||
|
||||
if [[ -z "${DIST}" || -z "${ARCH}" ]]; then
|
||||
echo "ERROR: Distro and Architecture must be specified." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
: ${CURL:=curl}
|
||||
|
||||
# ARTIFACTS_DIR represents the root of the artifacts (deb and rpm packages)
|
||||
# extracted from the packaging image.
|
||||
# TODO: accept ARTIFACTS_DIR as a command-line argument
|
||||
: "${ARTIFACTS_DIR="${PROJECT_ROOT}/artifacts"}"
|
||||
|
||||
@ -60,142 +52,68 @@ if [[ ! -d "${ARTIFACTS_DIR}" ]]; then
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
if [[ ! -f "${ARTIFACTS_DIR}/manifest.txt" ]]; then
|
||||
echo "ERROR: Manifest file not found." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
if [[ -z "${ARTIFACTORY_TOKEN}" ]]; then
|
||||
echo "ERROR: ARTIFACTORY_TOKEN must be defined." >&2
|
||||
assert_usage "$@"
|
||||
fi
|
||||
|
||||
# TODO: accept KITMACKER_DIR as a command-line argument
|
||||
# TODO: accept KITMAKER_DIR as a command-line argument
|
||||
: "${KITMAKER_DIR="${PROJECT_ROOT}/artifacts/kitmaker"}"
|
||||
|
||||
eval $(${SCRIPTS_DIR}/get-component-versions.sh)
|
||||
KITMAKER_SCRATCH="${KITMAKER_DIR}/.scratch"
|
||||
|
||||
# Returns the key=value property if the value isn't empty
|
||||
# Prepends with ";" if needed
|
||||
set_prop_value() {
|
||||
local key=$1
|
||||
local value=$2
|
||||
if [ -n "${value}" ]; then
|
||||
if [ -z "${PROPS}" ]; then
|
||||
echo "${key}=${value}"
|
||||
else
|
||||
echo ";${key}=${value}"
|
||||
fi
|
||||
fi
|
||||
# extract_info extracts the value of the specified variable from the manifest.txt file.
|
||||
function extract_info() {
|
||||
local variable=$1
|
||||
local value=$(cat "${ARTIFACTS_DIR}/manifest.txt" | grep "#${variable}" | sed -e "s/#${variable}=//" | tr -d '\r')
|
||||
echo $value
|
||||
}
|
||||
|
||||
process_props() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
IMAGE_EPOCH=$(extract_info "IMAGE_EPOCH")
|
||||
GIT_BRANCH=$(extract_info "GIT_BRANCH")
|
||||
GIT_COMMIT=$(extract_info "GIT_COMMIT")
|
||||
VERSION=$(extract_info "VERSION")
|
||||
|
||||
PROPS+=$(set_prop_value "component_name" "${COMPONENT_NAME}")
|
||||
PROPS+=$(set_prop_value "version" "${VERSION}")
|
||||
PROPS+=$(set_prop_value "os" "${dist}")
|
||||
PROPS+=$(set_prop_value "arch" "${arch}")
|
||||
PROPS+=$(set_prop_value "platform" "${dist}-${arch}")
|
||||
# TODO: Use `git describe` to get this information if it's not available.
|
||||
PROPS+=$(set_prop_value "changelist" "${CI_COMMIT_SHA}")
|
||||
PROPS+=$(set_prop_value "branch" "${CI_COMMIT_REF_NAME}")
|
||||
|
||||
# Gitlab variables to expose
|
||||
for var in CI_PROJECT_ID CI_PIPELINE_ID CI_JOB_ID CI_JOB_URL CI_PROJECT_PATH; do
|
||||
if [ -n "${!var}" ]; then
|
||||
PROPS+=$(set_prop_value "${var}" "${!var}")
|
||||
fi
|
||||
done
|
||||
# add_distro adds the specified component, os, and arch to the .package folder from which a kitmaker archive is generated.
|
||||
function add_distro() {
|
||||
local component=$1
|
||||
local os=$2
|
||||
local arch=$3
|
||||
|
||||
echo "Applying properties: ${PROPS}"
|
||||
}
|
||||
local package_dist=$4
|
||||
local package_arch=$5
|
||||
|
||||
## NOT USED:
|
||||
## can substitute this function place of upload_file to modify properties of
|
||||
## existing file instead of uploading files.
|
||||
# Sets the properties on a path
|
||||
# Relies on global variables: ARTIFACTORY_TOKEN, ARTIFACTORY_URL
|
||||
set_props() {
|
||||
local dist="$1"
|
||||
local arch="$2"
|
||||
local kitmakerfilename="$3"
|
||||
local name="${component}-${os}-${arch}"
|
||||
|
||||
# extract the Artifactory hostname
|
||||
artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
|
||||
local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/${kitmakerfilename}"
|
||||
|
||||
local PROPS
|
||||
process_props "${DIST}" "${ARCH}"
|
||||
|
||||
echo "Setting ${image_path} with properties: ${PROPS}"
|
||||
if ! ${CURL} -fs -H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
|
||||
-X PUT \
|
||||
"https://${artifactory_host}/artifactory/api/storage/${image_path}?properties=${PROPS}&recursive=0" ; then
|
||||
echo "ERROR: set props failed: ${image_path}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Uploads file to ARTIFACTS_DIR/<os>/<arch>/<filename>
|
||||
# Relies on global variables: DIST, ARCH, ARTIFACTORY_TOKEN, ARTIFACTORY_URL
|
||||
upload_file() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
local file=$3
|
||||
|
||||
# extract the Artifactory hostname
|
||||
artifactory_host=$(echo "${ARTIFACTORY_URL##https://}" | awk -F'/' '{print $1}')
|
||||
# get base part of the ARTIFACTORY_URL without hostname
|
||||
local image_path="${ARTIFACTORY_URL#https://${artifactory_host}/}/${dist}/${arch}/$(basename ${file})"
|
||||
|
||||
local PROPS
|
||||
process_props "${dist}" "${arch}"
|
||||
|
||||
if [ ! -r "${file}" ]; then
|
||||
echo "ERROR: File not found or not readable: ${file}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Collect sum
|
||||
SHA1_SUM=$(sha1sum -b "${file}" | awk '{ print $1 }')
|
||||
|
||||
echo "Uploading ${image_path} from ${file}"
|
||||
if ! ${CURL} -f \
|
||||
-H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
|
||||
-H "X-Checksum-Sha1: ${SHA1_SUM}" \
|
||||
${file:+-T ${file}} -X PUT \
|
||||
"https://${artifactory_host}/${image_path};${PROPS}" ;
|
||||
then
|
||||
echo "ERROR: upload file failed: ${file}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
function push-kitmaker-artifactory() {
|
||||
local dist=$1
|
||||
local arch=$2
|
||||
local archive=$3
|
||||
|
||||
upload_file "${dist}" "${arch}" "${archive}"
|
||||
}
|
||||
|
||||
# kitmakerize-distro creates a tar.gz archive for the specified dist-arch combination.
|
||||
# The archive is created at ${KITMAKER_DIR}/${name}.tar.gz (where ${name} is the third positional argument)
|
||||
function kitmakerize-distro() {
|
||||
local dist="$1"
|
||||
local arch="$2"
|
||||
local archive="$3"
|
||||
|
||||
local name=$(basename "${archive%%.tar.gz}")
|
||||
## Copy packages into directory layout for .tar.gz
|
||||
# TODO: make scratch_dir configurable
|
||||
local scratch_dir="$(dirname ${archive})/.scratch/${name}"
|
||||
local packages_dir="${scratch_dir}/.packages/"
|
||||
local scratch_dir="${KITMAKER_SCRATCH}/${name}"
|
||||
local packages_dir="${scratch_dir}/.packages"
|
||||
|
||||
mkdir -p "${packages_dir}"
|
||||
|
||||
# Copy the extracted files to the .packages directory so that a kitmaker file can be created.
|
||||
source="${ARTIFACTS_DIR}/packages/${dist}/${arch}"
|
||||
source="${ARTIFACTS_DIR}/packages/${package_dist}/${package_arch}"
|
||||
cp -r "${source}/"* "${packages_dir}/"
|
||||
}
|
||||
|
||||
# create_archive creates a kitmaker archive for the specified component, os, and arch.
|
||||
function create_archive() {
|
||||
local component=$1
|
||||
local os=$2
|
||||
local arch=$3
|
||||
local version=$4
|
||||
|
||||
local name="${component}-${os}-${arch}"
|
||||
local archive="${KITMAKER_DIR}/${name}-${version}.tar.gz"
|
||||
|
||||
local scratch_dir="${KITMAKER_SCRATCH}/${name}"
|
||||
local packages_dir="${scratch_dir}/.packages/"
|
||||
|
||||
## Tar up the directory structure created above
|
||||
tar zcvf "${archive}" -C "${scratch_dir}/.." "${name}"
|
||||
echo "Created: ${archive}"
|
||||
ls -l "${archive}"
|
||||
@ -209,8 +127,99 @@ function kitmakerize-distro() {
|
||||
rmdir "${scratch_dir}"
|
||||
}
|
||||
|
||||
: "${VERSION=$({NVIDIA_CONTAINER_TOOLKIT_PACKAGE_VERSION})}"
|
||||
kitmaker_name="${COMPONENT_NAME//-/_}-${DIST}-${ARCH}-${VERSION}"
|
||||
kitmaker_archive="${KITMAKER_DIR}/${kitmaker_name}.tar.gz"
|
||||
kitmakerize-distro "${DIST}" "${ARCH}" "${kitmaker_archive}"
|
||||
push-kitmaker-artifactory "${DIST}" "${ARCH}" "${kitmaker_archive}"
|
||||
function join_by { local IFS="$1"; shift; echo "$*"; }
|
||||
|
||||
function optionally_add_property() {
|
||||
local property=$1
|
||||
local value=$2
|
||||
if [[ -n "${value}" ]]; then
|
||||
props+=("${property}=${value}")
|
||||
fi
|
||||
}
|
||||
|
||||
function upload_archive() {
|
||||
local component=$1
|
||||
local os=$2
|
||||
local arch=$3
|
||||
local version=$4
|
||||
|
||||
local package_builds=$(join_by , ${@:5})
|
||||
|
||||
local name="${component}-${os}-${arch}"
|
||||
local archive="${KITMAKER_DIR}/${name}-${version}.tar.gz"
|
||||
|
||||
if [ ! -r "${archive}" ]; then
|
||||
echo "ERROR: File not found or not readable: ${archive}"
|
||||
exit 1
|
||||
fi
|
||||
local sha1_checksum=$(sha1sum -b "${archive}" | awk '{ print $1 }')
|
||||
|
||||
local upload_url="${KITMAKER_ARTIFACTORY_REPO}/${component}-${GIT_BRANCH}/default/$(basename ${archive})"
|
||||
|
||||
local props=()
|
||||
# Required KITMAKER properties:
|
||||
props+=("component_name=${component}")
|
||||
props+=("version=${version}")
|
||||
props+=("os=${os}")
|
||||
props+=("arch=${arch}")
|
||||
props+=("platform=${os}-${arch}")
|
||||
# TODO: extract the GIT commit from the packaging image
|
||||
props+=("changelist=${GIT_COMMIT}")
|
||||
props+=("branch=${GIT_BRANCH}")
|
||||
# Package properties:
|
||||
props+=("package.epoch=${IMAGE_EPOCH}")
|
||||
props+=("package.version=${VERSION}")
|
||||
optionally_add_property "package.builds" "${package_builds}"
|
||||
|
||||
for var in "CI_PROJECT_ID" "CI_PIPELINE_ID" "CI_JOB_ID" "CI_JOB_URL" "CI_PROJECT_PATH"; do
|
||||
if [ -n "${!var}" ]; then
|
||||
optionally_add_property "${var}" "${!var}"
|
||||
fi
|
||||
done
|
||||
local PROPS=$(join_by ";" "${props[@]}")
|
||||
|
||||
echo "Uploading ${upload_url} from ${file}"
|
||||
if ! ${CURL} -f \
|
||||
-H "X-JFrog-Art-Api: ${ARTIFACTORY_TOKEN}" \
|
||||
-H "X-Checksum-Sha1: ${sha1_checksum}" \
|
||||
${archive:+-T ${archive}} -X PUT \
|
||||
"${upload_url};${PROPS}" ;
|
||||
then
|
||||
echo "ERROR: upload file failed: ${archive}"
|
||||
exit 1
|
||||
fi
|
||||
}
|
||||
|
||||
component="nvidia_container_toolkit"
|
||||
version="${VERSION%-rc.*}"
|
||||
version_suffix=$(date -r "${IMAGE_EPOCH}" '+%Y.%m.%d.%s' || date -d @"${IMAGE_EPOCH}" '+%Y.%m.%d.%s')
|
||||
kitmaker_version="${VERSION%-rc.*}.${version_suffix}"
|
||||
kitmaker_os="linux"
|
||||
|
||||
# create_and_upload creates a kitmaker archive for the specified component, os, and arch and uploads it.
|
||||
function create_and_upload() {
|
||||
local kitmaker_arch=$1
|
||||
local builds=${@:2}
|
||||
|
||||
for build in ${builds}; do
|
||||
local package_dist=$(echo ${build} | cut -d- -f1)
|
||||
local package_arch=$(echo ${build} | cut -d- -f2)
|
||||
|
||||
add_distro "${component}" "${kitmaker_os}" "${kitmaker_arch}" "${package_dist}" "${package_arch}"
|
||||
done
|
||||
|
||||
create_archive "${component}" "${kitmaker_os}" "${kitmaker_arch}" "${kitmaker_version}"
|
||||
upload_archive "${component}" "${kitmaker_os}" "${kitmaker_arch}" "${kitmaker_version}" ${builds}
|
||||
}
|
||||
|
||||
# Create archive for x86_64 linux distributions
|
||||
create_and_upload "x86_64" "ubuntu18.04-amd64" "centos8-x86_64"
|
||||
|
||||
# Create archive for sbsa linux distributions
|
||||
create_and_upload "sbsa" "ubuntu18.04-arm64" "centos8-aarch64"
|
||||
# Create archive for aarch64 linux distributions
|
||||
# NOTE: From the perspective of the NVIDIA Container Toolkit aarch64 is just a duplicate of sbsa
|
||||
create_and_upload "aarch64" "ubuntu18.04-arm64" "centos8-aarch64"
|
||||
|
||||
# Create archive for ppc64le linux distributions
|
||||
create_and_upload "ppc64le" "ubuntu18.04-ppc64le" "centos8-ppc64le"
|
||||
|
||||
@ -29,3 +29,5 @@ CUDA_VERSION := 11.8.0
|
||||
GOLANG_VERSION := 1.18.8
|
||||
|
||||
GIT_COMMIT ?= $(shell git describe --match="" --dirty --long --always --abbrev=40 2> /dev/null || echo "")
|
||||
GIT_BRANCH ?= $(git rev-parse --abbrev-ref HEAD /dev/null || echo "${GIT_COMMIT}")
|
||||
SOURCE_DATE_EPOCH ?= $(shell git log -1 --format=%ct 2> /dev/null || echo "")
|
||||
|
||||
Loading…
Reference in New Issue
Block a user