From 88debb8e347540d6f5b46ffda87259713e15fc2c Mon Sep 17 00:00:00 2001 From: Carlos Eduardo Arango Gutierrez Date: Sun, 28 Jan 2024 18:22:51 +0100 Subject: [PATCH 1/3] Use test-infra devel image Signed-off-by: Carlos Eduardo Arango Gutierrez --- Makefile | 31 ++++++------------------------- versions.mk | 3 +++ 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/Makefile b/Makefile index 3afbd9c6..922e4445 100644 --- a/Makefile +++ b/Makefile @@ -100,31 +100,12 @@ coverage: test generate: go generate $(MODULE)/... -# Generate an image for containerized builds -# Note: This image is local only -.PHONY: .build-image .pull-build-image .push-build-image -.build-image: docker/Dockerfile.devel - if [ x"$(SKIP_IMAGE_BUILD)" = x"" ]; then \ - $(DOCKER) build \ - --progress=plain \ - --build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \ - --tag $(BUILDIMAGE) \ - -f $(^) \ - docker; \ - fi - -.pull-build-image: - $(DOCKER) pull $(BUILDIMAGE) - -.push-build-image: - $(DOCKER) push $(BUILDIMAGE) - -$(DOCKER_TARGETS): docker-%: .build-image - @echo "Running 'make $(*)' in docker container $(BUILDIMAGE)" +$(DOCKER_TARGETS): docker-%: + @echo "Running 'make $(*)' in container image $(BUILDIMAGE)" $(DOCKER) run \ --rm \ - -e GOCACHE=/tmp/.cache \ - -e GOLANGCI_LINT_CACHE=/tmp/.cache \ + -e GOCACHE=/tmp/.cache/go \ + -e GOMODCACHE=/tmp/.cache/gomod \ -v $(PWD):/work \ -w /work \ --user $$(id -u):$$(id -g) \ @@ -137,8 +118,8 @@ PHONY: .shell $(DOCKER) run \ --rm \ -ti \ - -e GOCACHE=/tmp/.cache \ - -e GOLANGCI_LINT_CACHE=/tmp/.cache \ + -e GOCACHE=/tmp/.cache/go \ + -e GOMODCACHE=/tmp/.cache/gomod \ -v $(PWD):/work \ -w /work \ --user $$(id -u):$$(id -g) \ diff --git a/versions.mk b/versions.mk index 65e2af5a..3b2fae45 100644 --- a/versions.mk +++ b/versions.mk @@ -33,6 +33,9 @@ LIBNVIDIA_CONTAINER0_VERSION := 0.10.0+jetpack CUDA_VERSION := 12.3.1 GOLANG_VERSION := 1.20.5 +BUILDIMAGE_TAG ?= devel-go$(GOLANG_VERSION) +BUILDIMAGE ?= ghcr.io/nvidia/k8s-test-infra:$(BUILDIMAGE_TAG) + GIT_COMMIT ?= $(shell git describe --match="" --dirty --long --always --abbrev=40 2> /dev/null || echo "") GIT_COMMIT_SHORT ?= $(shell git rev-parse --short HEAD 2> /dev/null || echo "") GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD 2> /dev/null || echo "${GIT_COMMIT}") From 195e3a22b4d0a932c49b77187f61bcc803f68942 Mon Sep 17 00:00:00 2001 From: Carlos Eduardo Arango Gutierrez Date: Sun, 28 Jan 2024 18:23:14 +0100 Subject: [PATCH 2/3] Drop go checks from gitlab ci Signed-off-by: Carlos Eduardo Arango Gutierrez --- .common-ci.yml | 1 - .gitlab-ci.yml | 37 +------------------------------------ 2 files changed, 1 insertion(+), 37 deletions(-) diff --git a/.common-ci.yml b/.common-ci.yml index 04f648ae..4a058764 100644 --- a/.common-ci.yml +++ b/.common-ci.yml @@ -19,7 +19,6 @@ default: variables: GIT_SUBMODULE_STRATEGY: recursive - BUILDIMAGE: "${CI_REGISTRY_IMAGE}/build:${CI_COMMIT_SHORT_SHA}" BUILD_MULTI_ARCH_IMAGES: "true" stages: diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 2b2b5dfe..86d0ec29 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -15,42 +15,8 @@ include: - .common-ci.yml -build-dev-image: - stage: image - script: - - apk --no-cache add make bash - - make .build-image - - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}" - - make .push-build-image - -.requires-build-image: - image: "${BUILDIMAGE}" - needs: - - job: build-dev-image - -check: - extends: - - .requires-build-image - stage: go-checks - script: - - make check - -go-build: - extends: - - .requires-build-image - stage: go-build - script: - - make build - -unit-tests: - extends: - - .requires-build-image - stage: unit-tests - script: - - make coverage - # Define the package build helpers -.multi-arch-build: + .multi-arch-build: before_script: - apk add --no-cache coreutils build-base sed git bash make - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes' @@ -288,4 +254,3 @@ test-docker-ubuntu20.04: - .dist-ubuntu20.04 needs: - image-ubuntu20.04 - From 6857f538a622808825cfe149b8f619df358bd271 Mon Sep 17 00:00:00 2001 From: Carlos Eduardo Arango Gutierrez Date: Sun, 28 Jan 2024 18:23:35 +0100 Subject: [PATCH 3/3] Add github actions Signed-off-by: Carlos Eduardo Arango Gutierrez --- .github/dependabot.yml | 20 ++++++ .github/workflows/blossom-ci.yml | 113 ------------------------------- .github/workflows/golang.yaml | 56 +++++++++++++++ .github/workflows/image.yaml | 108 +++++++++++++++++++++++++++++ .github/workflows/pre-sanity.yml | 22 ------ .gitlab-ci.yml | 2 +- 6 files changed, 185 insertions(+), 136 deletions(-) create mode 100644 .github/dependabot.yml delete mode 100644 .github/workflows/blossom-ci.yml create mode 100644 .github/workflows/golang.yaml create mode 100644 .github/workflows/image.yaml delete mode 100644 .github/workflows/pre-sanity.yml diff --git a/.github/dependabot.yml b/.github/dependabot.yml new file mode 100644 index 00000000..ee23ed2e --- /dev/null +++ b/.github/dependabot.yml @@ -0,0 +1,20 @@ +# Please see the documentation for all configuration options: +# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates + +version: 2 +updates: + - package-ecosystem: "gomod" + target-branch: main + directory: "/" + schedule: + interval: "weekly" + day: "sunday" + ignore: + - dependency-name: k8s.io/* + labels: + - dependencies + + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" diff --git a/.github/workflows/blossom-ci.yml b/.github/workflows/blossom-ci.yml deleted file mode 100644 index a5b37eb4..00000000 --- a/.github/workflows/blossom-ci.yml +++ /dev/null @@ -1,113 +0,0 @@ -# Copyright (c) 2020-2023, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# A workflow to trigger ci on hybrid infra (github + self hosted runner) -name: Blossom-CI -on: - issue_comment: - types: [created] - workflow_dispatch: - inputs: - platform: - description: 'runs-on argument' - required: false - args: - description: 'argument' - required: false -jobs: - Authorization: - name: Authorization - runs-on: blossom - outputs: - args: ${{ env.args }} - - # This job only runs for pull request comments - if: | - contains( '\ - anstockatnv,\ - rorajani,\ - cdesiniotis,\ - shivamerla,\ - ArangoGutierrez,\ - elezar,\ - klueska,\ - zvonkok,\ - ', format('{0},', github.actor)) && - github.event.comment.body == '/blossom-ci' - steps: - - name: Check if comment is issued by authorized person - run: blossom-ci - env: - OPERATION: 'AUTH' - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }} - - Vulnerability-scan: - name: Vulnerability scan - needs: [Authorization] - runs-on: ubuntu-latest - steps: - - name: Checkout code - uses: actions/checkout@v2 - with: - repository: ${{ fromJson(needs.Authorization.outputs.args).repo }} - ref: ${{ fromJson(needs.Authorization.outputs.args).ref }} - lfs: 'true' - - # repo specific steps - #- name: Setup java - # uses: actions/setup-java@v1 - # with: - # java-version: 1.8 - - # add blackduck properties https://synopsys.atlassian.net/wiki/spaces/INTDOCS/pages/631308372/Methods+for+Configuring+Analysis#Using-a-configuration-file - #- name: Setup blackduck properties - # run: | - # PROJECTS=$(mvn -am dependency:tree | grep maven-dependency-plugin | awk '{ out="com.nvidia:"$(NF-1);print out }' | grep rapids | xargs | sed -e 's/ /,/g') - # echo detect.maven.build.command="-pl=$PROJECTS -am" >> application.properties - # echo detect.maven.included.scopes=compile >> application.properties - - - name: Run blossom action - uses: NVIDIA/blossom-action@main - env: - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - REPO_KEY_DATA: ${{ secrets.BLOSSOM_KEY }} - with: - args1: ${{ fromJson(needs.Authorization.outputs.args).args1 }} - args2: ${{ fromJson(needs.Authorization.outputs.args).args2 }} - args3: ${{ fromJson(needs.Authorization.outputs.args).args3 }} - - Job-trigger: - name: Start ci job - needs: [Vulnerability-scan] - runs-on: blossom - steps: - - name: Start ci job - run: blossom-ci - env: - OPERATION: 'START-CI-JOB' - CI_SERVER: ${{ secrets.CI_SERVER }} - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - Upload-Log: - name: Upload log - runs-on: blossom - if : github.event_name == 'workflow_dispatch' - steps: - - name: Jenkins log for pull request ${{ fromJson(github.event.inputs.args).pr }} (click here) - run: blossom-ci - env: - OPERATION: 'POST-PROCESSING' - CI_SERVER: ${{ secrets.CI_SERVER }} - REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/golang.yaml b/.github/workflows/golang.yaml new file mode 100644 index 00000000..f29ecfcd --- /dev/null +++ b/.github/workflows/golang.yaml @@ -0,0 +1,56 @@ +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Golang + +on: + pull_request: + branches: + - main + - release-* + push: + branches: + - main + - release-* + +jobs: + check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Lint + uses: golangci/golangci-lint-action@v3 + with: + version: latest + args: -v --timeout 5m + skip-cache: true + test: + name: Unit test + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v4 + - name: Install Go + uses: actions/setup-go@v5 + with: + go-version: '1.20' + - run: make test + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Build + run: make docker-build diff --git a/.github/workflows/image.yaml b/.github/workflows/image.yaml new file mode 100644 index 00000000..588c6614 --- /dev/null +++ b/.github/workflows/image.yaml @@ -0,0 +1,108 @@ +# Copyright 2024 NVIDIA CORPORATION +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Run this workflow on pull requests +name: image + +on: + push: + branches: + - main + - release-* + +jobs: + packages: + runs-on: ubuntu-latest + strategy: + matrix: + package: + - ubuntu18.04-arm64 + - ubuntu18.04-amd64 + - ubuntu18.04-ppc64le + - centos7-aarch64 + - centos7-x86_64 + - centos8-ppc64le + fail-fast: false + steps: + - uses: actions/checkout@v4 + name: Check out code + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: build ${{ matrix.package }} packages + run: | + sudo apt-get install -y coreutils build-essential sed git bash make + echo "Building packages" + ./scripts/build-packages.sh ${{ matrix.package }} + - name: 'Upload Artifacts' + uses: actions/upload-artifact@v4 + with: + compression-level: 0 + name: toolkit-container-${{ matrix.package }}-${{ github.run_id }} + path: ${{ github.workspace }}/dist/* + + image: + runs-on: ubuntu-latest + strategy: + matrix: + image: [ubuntu20.04, centos7, ubi8, packaging] + needs: packages + steps: + - uses: actions/checkout@v4 + name: Check out code + - name: Calculate build vars + id: vars + run: | + echo "COMMIT_SHORT_SHA=${GITHUB_SHA:0:8}" >> $GITHUB_ENV + echo "LOWERCASE_REPO_OWNER=$(echo "${GITHUB_REPOSITORY_OWNER}" | awk '{print tolower($0)}')" >> $GITHUB_ENV + REPO_FULL_NAME="${{ github.event.pull_request.head.repo.full_name }}" + echo "${REPO_FULL_NAME}" + echo "LABEL_IMAGE_SOURCE=https://github.com/${REPO_FULL_NAME}" >> $GITHUB_ENV + + GENERATE_ARTIFACTS="false" + if [[ "${{ github.actor }}" == "dependabot[bot]" ]]; then + GENERATE_ARTIFACTS="false" + elif [[ "${{ github.event_name }}" == "pull_request" && "${{ github.event.pull_request.head.repo.full_name }}" == "${{ github.repository }}" ]]; then + GENERATE_ARTIFACTS="true" + elif [[ "${{ github.event_name }}" == "push" ]]; then + GENERATE_ARTIFACTS="true" + fi + echo "PUSH_ON_BUILD=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV + echo "BUILD_MULTI_ARCH_IMAGES=${GENERATE_ARTIFACTS}" >> $GITHUB_ENV + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + - name: Get built packages + uses: actions/download-artifact@v4 + with: + path: ${{ github.workspace }}/dist/ + pattern: toolkit-container-* + merge-multiple: true + + - name: Login to GitHub Container Registry + uses: docker/login-action@v3 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build image + env: + IMAGE_NAME: ghcr.io/${LOWERCASE_REPO_OWNER}/container-toolkit + VERSION: ${COMMIT_SHORT_SHA} + run: | + echo "${VERSION}" + make -f build/container/Makefile build-${{ matrix.image }} diff --git a/.github/workflows/pre-sanity.yml b/.github/workflows/pre-sanity.yml deleted file mode 100644 index 7e2ef582..00000000 --- a/.github/workflows/pre-sanity.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: Run pre sanity - -# run this workflow for each commit -on: [pull_request] - -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - - name: Build dev image - run: make .build-image - - - name: Build - run: make docker-build - - - name: Tests - run: make docker-coverage - - - name: Checks - run: make docker-check diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 86d0ec29..29159ea2 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -16,7 +16,7 @@ include: - .common-ci.yml # Define the package build helpers - .multi-arch-build: +.multi-arch-build: before_script: - apk add --no-cache coreutils build-base sed git bash make - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'