mirror of
https://github.com/NVIDIA/nvidia-container-toolkit
synced 2024-12-04 01:55:05 +00:00
306 lines
9.1 KiB
YAML
306 lines
9.1 KiB
YAML
# Copyright (c) 2021-2022, NVIDIA CORPORATION. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
include:
|
|
- local: '.common-ci.yml'
|
|
|
|
default:
|
|
tags:
|
|
- cnt
|
|
- container-dev
|
|
- docker/multi-arch
|
|
- docker/privileged
|
|
- os/linux
|
|
- type/docker
|
|
|
|
variables:
|
|
DOCKER_DRIVER: overlay2
|
|
DOCKER_TLS_CERTDIR: "/certs"
|
|
# Release "devel"-tagged images off the main branch
|
|
RELEASE_DEVEL_BRANCH: "main"
|
|
DEVEL_RELEASE_IMAGE_VERSION: "devel"
|
|
# On the multi-arch builder we don't need the qemu setup.
|
|
SKIP_QEMU_SETUP: "1"
|
|
# Define the public staging registry
|
|
STAGING_REGISTRY: ghcr.io/nvidia
|
|
STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
|
|
ARTIFACTORY_REPO_BASE: "https://urm.nvidia.com/artifactory/sw-gpu-cloudnative"
|
|
KITMAKER_RELEASE_FOLDER: "kitmaker"
|
|
PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
|
|
|
|
.image-pull:
|
|
stage: image-build
|
|
variables:
|
|
IN_REGISTRY: "${STAGING_REGISTRY}"
|
|
IN_IMAGE_NAME: container-toolkit
|
|
IN_VERSION: "${STAGING_VERSION}"
|
|
OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
|
|
OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
|
|
OUT_REGISTRY: "${CI_REGISTRY}"
|
|
OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
|
|
PUSH_MULTIPLE_TAGS: "false"
|
|
# We delay the job start to allow the public pipeline to generate the required images.
|
|
rules:
|
|
- when: delayed
|
|
start_in: 30 minutes
|
|
timeout: 30 minutes
|
|
retry:
|
|
max: 2
|
|
when:
|
|
- job_execution_timeout
|
|
- stuck_or_timeout_failure
|
|
before_script:
|
|
- !reference [.regctl-setup, before_script]
|
|
- apk add --no-cache make bash
|
|
- >
|
|
regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
|
|
script:
|
|
- regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
|
|
- make -f deployments/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
|
|
|
|
image-ubi8:
|
|
extends:
|
|
- .dist-ubi8
|
|
- .image-pull
|
|
|
|
image-ubuntu20.04:
|
|
extends:
|
|
- .dist-ubuntu20.04
|
|
- .image-pull
|
|
|
|
# The DIST=packaging target creates an image containing all built packages
|
|
image-packaging:
|
|
extends:
|
|
- .dist-packaging
|
|
- .image-pull
|
|
|
|
# We skip the integration tests for the internal CI:
|
|
.integration:
|
|
stage: test
|
|
before_script:
|
|
- echo "Skipped in internal CI"
|
|
script:
|
|
- echo "Skipped in internal CI"
|
|
|
|
# The .scan step forms the base of the image scan operation performed before releasing
|
|
# images.
|
|
.scan:
|
|
stage: scan
|
|
image: "${PULSE_IMAGE}"
|
|
variables:
|
|
IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
|
|
IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
|
|
rules:
|
|
- if: $SKIP_SCANS != "yes"
|
|
- when: manual
|
|
before_script:
|
|
- docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
|
|
# TODO: We should specify the architecture here and scan all architectures
|
|
- docker pull --platform="${PLATFORM}" "${IMAGE}"
|
|
- docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
|
|
- AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
|
|
- >
|
|
export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" | tr -d '"')
|
|
- if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
|
|
script:
|
|
- pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
|
|
- rm -f "${IMAGE_ARCHIVE}"
|
|
artifacts:
|
|
when: always
|
|
expire_in: 1 week
|
|
paths:
|
|
- pulse-cli.log
|
|
- licenses.json
|
|
- sbom.json
|
|
- vulns.json
|
|
- policy_evaluation.json
|
|
|
|
# Define the scan targets
|
|
scan-ubuntu20.04-amd64:
|
|
extends:
|
|
- .dist-ubuntu20.04
|
|
- .platform-amd64
|
|
- .scan
|
|
needs:
|
|
- image-ubuntu20.04
|
|
|
|
scan-ubuntu20.04-arm64:
|
|
extends:
|
|
- .dist-ubuntu20.04
|
|
- .platform-arm64
|
|
- .scan
|
|
needs:
|
|
- image-ubuntu20.04
|
|
- scan-ubuntu20.04-amd64
|
|
|
|
scan-ubi8-amd64:
|
|
extends:
|
|
- .dist-ubi8
|
|
- .platform-amd64
|
|
- .scan
|
|
needs:
|
|
- image-ubi8
|
|
|
|
scan-ubi8-arm64:
|
|
extends:
|
|
- .dist-ubi8
|
|
- .platform-arm64
|
|
- .scan
|
|
needs:
|
|
- image-ubi8
|
|
- scan-ubi8-amd64
|
|
|
|
scan-packaging:
|
|
extends:
|
|
- .dist-packaging
|
|
- .scan
|
|
needs:
|
|
- image-packaging
|
|
|
|
# Define external release helpers
|
|
.release:ngc:
|
|
extends:
|
|
- .release:external
|
|
variables:
|
|
OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
|
|
OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
|
|
OUT_REGISTRY: "${NGC_REGISTRY}"
|
|
OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
|
|
|
|
.release:packages:
|
|
stage: release
|
|
needs:
|
|
- image-packaging
|
|
variables:
|
|
VERSION: "${CI_COMMIT_SHORT_SHA}"
|
|
PACKAGE_REGISTRY: "${CI_REGISTRY}"
|
|
PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
|
|
PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
|
|
PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
|
|
PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
|
|
KITMAKER_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${KITMAKER_RELEASE_FOLDER}"
|
|
ARTIFACTS_DIR: "${CI_PROJECT_DIR}/artifacts"
|
|
script:
|
|
- !reference [.regctl-setup, before_script]
|
|
- apk add --no-cache bash git
|
|
- regctl registry login "${PACKAGE_REGISTRY}" -u "${PACKAGE_REGISTRY_USER}" -p "${PACKAGE_REGISTRY_TOKEN}"
|
|
- ./scripts/extract-packages.sh "${PACKAGE_IMAGE_NAME}:${PACKAGE_IMAGE_TAG}"
|
|
- ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
|
|
- rm -rf ${ARTIFACTS_DIR}
|
|
|
|
# Define the package release targets
|
|
release:packages:kitmaker:
|
|
extends:
|
|
- .release:packages
|
|
|
|
release:archive:
|
|
extends:
|
|
- .release:external
|
|
needs:
|
|
- image-packaging
|
|
variables:
|
|
VERSION: "${CI_COMMIT_SHORT_SHA}"
|
|
PACKAGE_REGISTRY: "${CI_REGISTRY}"
|
|
PACKAGE_REGISTRY_USER: "${CI_REGISTRY_USER}"
|
|
PACKAGE_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
|
|
PACKAGE_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
|
|
PACKAGE_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}-packaging"
|
|
PACKAGE_ARCHIVE_ARTIFACTORY_REPO: "${ARTIFACTORY_REPO_BASE}-generic-local/${PACKAGE_ARCHIVE_RELEASE_FOLDER}"
|
|
script:
|
|
- apk add --no-cache bash git
|
|
- ./scripts/archive-packages.sh "${PACKAGE_ARCHIVE_ARTIFACTORY_REPO}"
|
|
|
|
release:staging-ubuntu20.04:
|
|
extends:
|
|
- .release:staging
|
|
- .dist-ubuntu20.04
|
|
needs:
|
|
- image-ubuntu20.04
|
|
|
|
# Define the external release targets
|
|
# Release to NGC
|
|
release:ngc-ubuntu20.04:
|
|
extends:
|
|
- .dist-ubuntu20.04
|
|
- .release:ngc
|
|
|
|
release:ngc-ubi8:
|
|
extends:
|
|
- .dist-ubi8
|
|
- .release:ngc
|
|
|
|
release:ngc-packaging:
|
|
extends:
|
|
- .dist-packaging
|
|
- .release:ngc
|
|
|
|
# Define the external image signing steps for NGC
|
|
# Download the ngc cli binary for use in the sign steps
|
|
.ngccli-setup:
|
|
before_script:
|
|
- apt-get update && apt-get install -y curl unzip jq
|
|
- |
|
|
if [ -z "${NGCCLI_VERSION}" ]; then
|
|
NGC_VERSION_URL="https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions"
|
|
# Extract the latest version from the JSON data using jq
|
|
export NGCCLI_VERSION=$(curl -s $NGC_VERSION_URL | jq -r '.recipe.latestVersionIdStr')
|
|
fi
|
|
echo "NGCCLI_VERSION ${NGCCLI_VERSION}"
|
|
- curl -sSLo ngccli_linux.zip https://api.ngc.nvidia.com/v2/resources/nvidia/ngc-apps/ngc_cli/versions/${NGCCLI_VERSION}/files/ngccli_linux.zip
|
|
- unzip ngccli_linux.zip
|
|
- chmod u+x ngc-cli/ngc
|
|
|
|
# .sign forms the base of the deployment jobs which signs images in the CI registry.
|
|
# This is extended with the image name and version to be deployed.
|
|
.sign:ngc:
|
|
image: ubuntu:latest
|
|
stage: sign
|
|
rules:
|
|
- if: $CI_COMMIT_TAG
|
|
variables:
|
|
NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}"
|
|
IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
|
|
IMAGE_TAG: "${CI_COMMIT_TAG}-${DIST}"
|
|
retry:
|
|
max: 2
|
|
before_script:
|
|
- !reference [.ngccli-setup, before_script]
|
|
# We ensure that the IMAGE_NAME and IMAGE_TAG is set
|
|
- 'echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1'
|
|
- 'echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1'
|
|
script:
|
|
- 'echo "Signing the image ${IMAGE_NAME}:${IMAGE_TAG}"'
|
|
- ngc-cli/ngc registry image publish --source ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:${IMAGE_TAG} --public --discoverable --allow-guest --sign --org nvidia
|
|
|
|
sign:ngc-ubuntu20.04:
|
|
extends:
|
|
- .dist-ubuntu20.04
|
|
- .sign:ngc
|
|
needs:
|
|
- release:ngc-ubuntu20.04
|
|
|
|
sign:ngc-ubi8:
|
|
extends:
|
|
- .dist-ubi8
|
|
- .sign:ngc
|
|
needs:
|
|
- release:ngc-ubi8
|
|
|
|
sign:ngc-packaging:
|
|
extends:
|
|
- .dist-packaging
|
|
- .sign:ngc
|
|
needs:
|
|
- release:ngc-packaging
|