From cd1540300e8ff60b785c950689b88381807bc7a1 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 15 Oct 2021 14:03:37 +0200 Subject: [PATCH 1/2] Add internal CI definition for release Signed-off-by: Evan Lezar --- .common-ci.yml | 8 +- .nvidia-ci.yml | 167 ++++++++++++++++++++++++++++++++++++++ tools/container/README.md | 2 + 3 files changed, 173 insertions(+), 4 deletions(-) create mode 100644 .nvidia-ci.yml diff --git a/.common-ci.yml b/.common-ci.yml index 3a4ce97c..c252332e 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -138,13 +138,13 @@ unit-tests: .multi-arch-build: before_script: - apk add --no-cache coreutils build-base sed git bash make - - docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes + - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes' .package-artifacts: variables: - ARTIFACTS_NAME: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-${CI_JOB_NAME}-artifacts-${CI_PIPELINE_ID}" - ARTIFACTS_ROOT: "${CI_PROJECT_NAME}-${CI_COMMIT_REF_SLUG}-artifacts-${CI_PIPELINE_ID}" - DIST_DIR: "${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}" + ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}" + ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}" + DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT} .package-build: extends: diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml new file mode 100644 index 00000000..6b4e5e56 --- /dev/null +++ b/.nvidia-ci.yml @@ -0,0 +1,167 @@ +# Copyright (c) 2021, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +include: + - local: '.common-ci.yml' + +default: + tags: + - cnt + - container-dev + - docker/multi-arch + - docker/privileged + - os/linux + - type/docker + +variables: + DOCKER_DRIVER: overlay2 + DOCKER_TLS_CERTDIR: "/certs" + # Release "devel"-tagged images off the master branch + RELEASE_DEVEL_BRANCH: "master" + RELEASE_DEVEL_TAG: "devel" + # On the multi-arch builder we don't need the qemu setup. + SKIP_QEMU_SETUP: "1" + +# We skip the integration tests for the internal CI: +.integration: + stage: test + before_script: + - echo "Skipped in internal CI" + script: + - echo "Skipped in internal CI" + +# The .scan step forms the base of the image scan operation performed before releasing +# images. +.scan: + stage: scan + variables: + IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}" + # SCAN_IMAGE must be a local image + SCAN_IMAGE: "container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}" + rules: + - if: $CI_COMMIT_MESSAGE =~ /\[skip[ _-]scans?\]/i + when: never + - if: $SKIP_SCANS + when: never + - if: $CI_COMMIT_TAG == null && $CI_COMMIT_BRANCH != $RELEASE_DEVEL_BRANCH + allow_failure: true + before_script: + - apk add --no-cache git + - apk add --no-cache python3 python3-dev py3-pip py3-wheel libmagic + - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" + # TODO: We should specify the architecture here and scan all architectures + - docker pull "${IMAGE}" + - docker tag "${IMAGE}" "${SCAN_IMAGE}" + - git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@gitlab-master.nvidia.com/sectooling/scanning/contamer.git + - pip3 install -r contamer/requirements.txt + script: + - cd contamer + - python3 contamer.py -ls --fail-on-non-os ${CONTAMER_SUPPRESS_VULNS:+--suppress-vulns ${CONTAMER_SUPPRESS_VULNS}} -- "${SCAN_IMAGE}" + + +# Define the scan targets +scan-centos7: + extends: + - .scan + - .dist-centos7 + needs: + - image-centos7 + +scan-centos8: + extends: + - .scan + - .dist-centos8 + needs: + - image-centos8 + +scan-ubuntu18.04: + extends: + - .scan + - .dist-ubuntu18.04 + needs: + - image-ubuntu18.04 + +scan-ubi8: + extends: + - .scan + - .dist-ubi8 + needs: + - image-ubi8 + +# Define external release helpers +.release:ngc: + extends: + - .release:external + variables: + OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}" + OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}" + OUT_REGISTRY: "${NGC_REGISTRY}" + OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}" + # TODO: For now we disable external releases + DOCKER: echo + +.release:dockerhub: + extends: + - .release:external + variables: + OUT_REGISTRY_USER: "${REGISTRY_USER}" + OUT_REGISTRY_TOKEN: "${REGISTRY_TOKEN}" + OUT_REGISTRY: "${DOCKERHUB_REGISTRY}" + OUT_IMAGE_NAME: "${REGISTRY_IMAGE}" + + # TODO: For now we disable external releases + DOCKER: echo + +# Define the external release targets +# Release to NGC +release:ngc-centos7: + extends: + - .release:ngc + - .dist-centos7 + +release:ngc-centos8: + extends: + - .release:ngc + - .dist-centos8 + +release:ngc-ubuntu18: + extends: + - .release:ngc + - .dist-ubuntu18.04 + +release:ngc-ubi8: + extends: + - .release:ngc + - .dist-ubi8 + +# Release to Dockerhub +release:dockerhub-centos7: + extends: + - .release:dockerhub + - .dist-centos7 + +release:dockerhub-centos8: + extends: + - .release:dockerhub + - .dist-centos8 + +release:dockerhub-ubuntu18: + extends: + - .release:dockerhub + - .dist-ubuntu18.04 + +release:dockerhub-ubi8: + extends: + - .release:dockerhub + - .dist-ubi8 diff --git a/tools/container/README.md b/tools/container/README.md index 3197b358..eb95f107 100644 --- a/tools/container/README.md +++ b/tools/container/README.md @@ -4,6 +4,8 @@ This repository contains tools that allow docker, containerd, or cri-o to be con *Note*: These were copied from the [`container-config` repository](https://gitlab.com/nvidia/container-toolkit/container-config/-/tree/383587f766a55177ede0e39e3810a974043e503e) are being migrated to commands installed with the NVIDIA Container Toolkit. +These will be migrated into an upcoming `nvidia-ctk` CLI as required. + ### Docker After building the `docker` binary, run: From 5b1a6765c668df1c4eef469093c094ae18f2cbe3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 25 Oct 2021 12:05:51 +0200 Subject: [PATCH 2/2] Remove rule for merge requests Signed-off-by: Evan Lezar --- .gitlab-ci.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 3542d9b5..81d2abfb 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -31,8 +31,6 @@ include: rules: - if: $CI_COMMIT_TAG when: always - - if: $CI_MERGE_REQUEST_ID - when: always # The full set of build-all jobs organized to # have builds for each ARCH run in parallel.