From 7fb04878c73b091ce0a16afc93240021a626d942 Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Tue, 23 Nov 2021 20:59:52 +0100
Subject: [PATCH 1/8] Include all architecture packages in toolkit container

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 .common-ci.yml                    |  5 ++---
 build/container/Dockerfile.centos | 13 +++++++------
 build/container/Dockerfile.ubuntu | 13 +++++++------
 build/container/Makefile          | 13 +++++++++----
 4 files changed, 25 insertions(+), 19 deletions(-)

diff --git a/.common-ci.yml b/.common-ci.yml
index c252332e..bd820c0d 100644
--- a/.common-ci.yml
+++ b/.common-ci.yml
@@ -242,9 +242,8 @@ image-ubuntu18.04:
     - .dist-ubuntu18.04
   needs:
     - package-ubuntu18.04-amd64
-    # TODO: These will be required once we generate multi-arch images
-    # - package-ubuntu18.04-arm64
-    # - package-ubuntu18.04-ppc64le
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
   script:
     - make -f build/container/Makefile build-${DIST}
     - make -f build/container/Makefile push-${DIST}
diff --git a/build/container/Dockerfile.centos b/build/container/Dockerfile.centos
index b47778ec..35e65968 100644
--- a/build/container/Dockerfile.centos
+++ b/build/container/Dockerfile.centos
@@ -46,16 +46,17 @@ ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
+ARG ARTIFACTS_DIR
+COPY ${ARTIFACTS_DIR}/ /artifacts/packages/
+
 WORKDIR /artifacts/packages
 
-ARG ARTIFACTS_DIR
-COPY ${ARTIFACTS_DIR}/* /artifacts/packages/
-
 ARG PACKAGE_VERSION
+ARG PACKAGE_ARCH
 RUN yum localinstall -y \
-    libnvidia-container1-${PACKAGE_VERSION}*.rpm \
-    libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
-    nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
+    ${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
 
 WORKDIR /work
 
diff --git a/build/container/Dockerfile.ubuntu b/build/container/Dockerfile.ubuntu
index 6cd3bfa1..a1d610d9 100644
--- a/build/container/Dockerfile.ubuntu
+++ b/build/container/Dockerfile.ubuntu
@@ -52,16 +52,17 @@ ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
+ARG ARTIFACTS_DIR
+COPY ${ARTIFACTS_DIR}/ /artifacts/packages/
+
 WORKDIR /artifacts/packages
 
-ARG ARTIFACTS_DIR
-COPY ${ARTIFACTS_DIR}/* /artifacts/packages/
-
 ARG PACKAGE_VERSION
+ARG PACKAGE_ARCH
 RUN dpkg -i \
-    libnvidia-container1_${PACKAGE_VERSION}*.deb \
-    libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
-    nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
+    ${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
 
 WORKDIR /work
 
diff --git a/build/container/Makefile b/build/container/Makefile
index 4e6fb1ce..38c9064d 100644
--- a/build/container/Makefile
+++ b/build/container/Makefile
@@ -58,22 +58,26 @@ build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
 
 # Use a generic build target to build the relevant images
 $(BUILD_TARGETS): build-%: $(ARTIFACTS_DIR)
-	$(DOCKER) build --pull \
+	DOCKER_BUILDKIT=1 \
+		$(DOCKER) build --platform=linux/amd64 --pull \
 		--tag $(IMAGE) \
 		--build-arg ARTIFACTS_DIR="$(ARTIFACTS_DIR)" \
 		--build-arg BASE_DIST="$(BASE_DIST)" \
 		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
 		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
+		--build-arg PACKAGE_ARCH="$(PACKAGE_ARCH)" \
 		--build-arg VERSION="$(VERSION)" \
 		-f $(DOCKERFILE) \
 		$(CURDIR)
 
 
 ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
+ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)
+
+PACKAGE_ARCH = amd64
 
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
-build-ubuntu%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/amd64
 build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 
 build-ubuntu18.04: BASE_DIST := ubuntu18.04
@@ -81,13 +85,14 @@ build-ubuntu20.04: BASE_DIST := ubuntu20.04
 
 build-ubi8: DOCKERFILE_SUFFIX := centos
 # TODO: Update this to use the centos8 packages
-build-ubi8: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/centos7/x86_64
+build-ubi8: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/centos7
 build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 build-ubi8: BASE_DIST := ubi8
+build-ubi8: PACKAGE_ARCH := x86_64
 
 build-centos%: DOCKERFILE_SUFFIX := centos
-build-centos%: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)/x86_64
 build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
+build-centos%: PACKAGE_ARCH := x86_64
 
 build-centos7: BASE_DIST := centos7
 build-centos8: BASE_DIST := centos8

From 629d575fad2464a511d750bda9b54408f504c220 Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Thu, 25 Nov 2021 20:02:07 +0100
Subject: [PATCH 2/8] Add packaging target that includes all release packages

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 build/container/Dockerfile.packaging | 32 ++++++++++++++++++++++++++++
 build/container/Makefile             |  9 ++++++--
 2 files changed, 39 insertions(+), 2 deletions(-)
 create mode 100644 build/container/Dockerfile.packaging

diff --git a/build/container/Dockerfile.packaging b/build/container/Dockerfile.packaging
new file mode 100644
index 00000000..da1a4ce8
--- /dev/null
+++ b/build/container/Dockerfile.packaging
@@ -0,0 +1,32 @@
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG BASE_DIST
+ARG CUDA_VERSION
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
+
+ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
+
+ARG ARTIFACTS_DIR
+COPY ${ARTIFACTS_DIR}/ /artifacts/packages/
+
+WORKDIR /artifacts/packages
+
+COPY ./LICENSE /licenses/LICENSE
+
+# This container is not intended to be run
+ENTRYPOINT ["exit", "1"]
diff --git a/build/container/Makefile b/build/container/Makefile
index 38c9064d..f37880bb 100644
--- a/build/container/Makefile
+++ b/build/container/Makefile
@@ -31,14 +31,15 @@ IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
 
 ##### Public rules #####
 DEFAULT_PUSH_TARGET := ubuntu18.04
-TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
+TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8 packaging
 
 BUILD_TARGETS := $(patsubst %, build-%, $(TARGETS))
 PUSH_TARGETS := $(patsubst %, push-%, $(TARGETS))
-TEST_TARGETS := $(patsubst %, test-%, $(TARGETS))
+TEST_TARGETS := $(patsubst %, test-%, $(filter-out packaging,$(TARGETS)))
 
 .PHONY: $(TARGETS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
+push-%: DIST = $(*)
 $(PUSH_TARGETS): push-%:
 	$(DOCKER) push "$(IMAGE_NAME):$(IMAGE_TAG)"
 
@@ -97,6 +98,10 @@ build-centos%: PACKAGE_ARCH := x86_64
 build-centos7: BASE_DIST := centos7
 build-centos8: BASE_DIST := centos8
 
+build-packaging: BASE_DIST := ubuntu20.04
+build-packaging: DOCKERFILE_SUFFIX := packaging
+build-packaging: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)
+
 # Test targets
 test-%: DIST = $(*)
 

From 2964f265333ace6e82aa2b2ee02d020c6ec57176 Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Thu, 25 Nov 2021 20:02:31 +0100
Subject: [PATCH 3/8] Add packaging target to CI

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 .common-ci.yml | 170 ++++++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 146 insertions(+), 24 deletions(-)

diff --git a/.common-ci.yml b/.common-ci.yml
index bd820c0d..fa1c045f 100644
--- a/.common-ci.yml
+++ b/.common-ci.yml
@@ -98,6 +98,14 @@ unit-tests:
 
 
 # Define the distribution targets
+.dist-amazonlinux1:
+  variables:
+    DIST: amazonlinux1
+
+.dist-amazonlinux2:
+  variables:
+    DIST: amazonlinux2
+
 .dist-centos7:
   variables:
     DIST: centos7
@@ -106,14 +114,35 @@ unit-tests:
   variables:
     DIST: centos8
 
+.dist-debian10:
+  variables:
+    DIST: debian10
+
+.dist-debian9:
+  variables:
+    DIST: debian9
+
+.dist-opensuse-leap15.1:
+  variables:
+    DIST: opensuse-leap15.1
+
 .dist-ubi8:
   variables:
     DIST: ubi8
 
+.dist-ubuntu16.04:
+  variables:
+    DIST: ubuntu16.04
+
 .dist-ubuntu18.04:
   variables:
     DIST: ubuntu18.04
 
+.dist-packaging:
+  variables:
+    DIST: packaging
+
+# Define architecture targets
 .arch-aarch64:
   variables:
     ARCH: aarch64
@@ -160,6 +189,84 @@ unit-tests:
       - ${ARTIFACTS_ROOT}
 
 # Define the package build targets
+package-amazonlinux1-x86_64:
+  extends:
+    - .package-build
+    - .dist-amazonlinux1
+    - .arch-x86_64
+
+package-amazonlinux2-aarch64:
+  extends:
+    - .package-build
+    - .dist-amazonlinux2
+    - .arch-aarch64
+
+package-amazonlinux2-x86_64:
+  extends:
+    - .package-build
+    - .dist-amazonlinux2
+    - .arch-x86_64
+
+package-centos7-ppc64le:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-ppc64le
+
+package-centos7-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-x86_64
+
+package-centos8-aarch64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-aarch64
+
+package-centos8-ppc64le:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-ppc64le
+
+package-centos8-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-x86_64
+
+package-debian10-amd64:
+  extends:
+    - .package-build
+    - .dist-debian10
+    - .arch-amd64
+
+package-debian9-amd64:
+  extends:
+    - .package-build
+    - .dist-debian9
+    - .arch-amd64
+
+package-opensuse-leap15.1-x86_64:
+  extends:
+    - .package-build
+    - .dist-opensuse-leap15.1
+    - .arch-x86_64
+
+package-ubuntu16.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-amd64
+
+package-ubuntu16.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-ppc64le
+
 package-ubuntu18.04-amd64:
   extends:
     - .package-build
@@ -178,18 +285,6 @@ package-ubuntu18.04-ppc64le:
     - .dist-ubuntu18.04
     - .arch-ppc64le
 
-package-centos7-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-x86_64
-
-package-centos8-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-x86_64
-
 # Define the image build targets
 .image-build:
   stage: image-build
@@ -200,6 +295,9 @@ package-centos8-x86_64:
     - apk add --no-cache bash make
     - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
     - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
 
 image-centos7:
   extends:
@@ -207,10 +305,8 @@ image-centos7:
     - .package-artifacts
     - .dist-centos7
   needs:
+    - package-centos7-ppc64le
     - package-centos7-x86_64
-  script:
-    - make -f build/container/Makefile build-${DIST}
-    - make -f build/container/Makefile push-${DIST}
 
 image-centos8:
   extends:
@@ -218,10 +314,9 @@ image-centos8:
     - .package-artifacts
     - .dist-centos8
   needs:
+    - package-centos8-aarch64
     - package-centos8-x86_64
-  script:
-    - make -f build/container/Makefile build-${DIST}
-    - make -f build/container/Makefile push-${DIST}
+    - package-centos8-ppc64le
 
 image-ubi8:
   extends:
@@ -230,10 +325,8 @@ image-ubi8:
     - .dist-ubi8
   needs:
     # Note: The ubi8 image currently uses the centos7 packages
+    - package-centos7-ppc64le
     - package-centos7-x86_64
-  script:
-    - make -f build/container/Makefile build-${DIST}
-    - make -f build/container/Makefile push-${DIST}
 
 image-ubuntu18.04:
   extends:
@@ -244,9 +337,31 @@ image-ubuntu18.04:
     - package-ubuntu18.04-amd64
     - package-ubuntu18.04-arm64
     - package-ubuntu18.04-ppc64le
-  script:
-    - make -f build/container/Makefile build-${DIST}
-    - make -f build/container/Makefile push-${DIST}
+
+# The DIST=packaging target creates an image containing all built packages
+image-packaging:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-packaging
+  needs:
+    - package-amazonlinux1-x86_64
+    - package-amazonlinux2-aarch64
+    - package-amazonlinux2-x86_64
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
+    - package-centos8-aarch64
+    - package-centos8-ppc64le
+    - package-centos8-x86_64
+    - package-debian10-amd64
+    - package-debian9-amd64
+    - package-opensuse-leap15.1-x86_64
+    - package-ubuntu16.04-amd64
+    - package-ubuntu16.04-ppc64le
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
+
 
 # Define test helpers
 .integration:
@@ -406,3 +521,10 @@ release:staging-ubuntu18.04:
     - test-containerd-ubuntu18.04
     - test-crio-ubuntu18.04
     - test-docker-ubuntu18.04
+
+release:staging-packaging:
+  extends:
+    - .release:staging
+    - .dist-packaging
+  needs:
+    - image-packaging

From e662e8197c474f7ac78c10ae0a9f229ceb0e74e5 Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Thu, 25 Nov 2021 20:43:43 +0100
Subject: [PATCH 4/8] Add placeholder for testing packaging image

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 .common-ci.yml                       |  9 ++++++++-
 build/container/Dockerfile.packaging |  3 ---
 build/container/Makefile             | 20 ++++++++++++++++++++
 3 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/.common-ci.yml b/.common-ci.yml
index fa1c045f..3343bcc2 100644
--- a/.common-ci.yml
+++ b/.common-ci.yml
@@ -433,6 +433,13 @@ test-docker-ubuntu18.04:
   needs:
     - image-ubuntu18.04
 
+test-packaging:
+  extends:
+    - .integration
+    - .dist-packaging
+  needs:
+    - image-packaging
+
 # .release forms the base of the deployment jobs which push images to the CI registry.
 # This is extended with the version to be deployed (e.g. the SHA or TAG) and the
 # target os.
@@ -527,4 +534,4 @@ release:staging-packaging:
     - .release:staging
     - .dist-packaging
   needs:
-    - image-packaging
+    - test-packaging
diff --git a/build/container/Dockerfile.packaging b/build/container/Dockerfile.packaging
index da1a4ce8..444109c9 100644
--- a/build/container/Dockerfile.packaging
+++ b/build/container/Dockerfile.packaging
@@ -27,6 +27,3 @@ COPY ${ARTIFACTS_DIR}/ /artifacts/packages/
 WORKDIR /artifacts/packages
 
 COPY ./LICENSE /licenses/LICENSE
-
-# This container is not intended to be run
-ENTRYPOINT ["exit", "1"]
diff --git a/build/container/Makefile b/build/container/Makefile
index f37880bb..20893a49 100644
--- a/build/container/Makefile
+++ b/build/container/Makefile
@@ -112,3 +112,23 @@ $(TEST_TARGETS): test-%:
 		$(IMAGE) \
 			--no-cleanup-on-error
 
+.PHONY: test-packaging
+test-packaging: DIST = packaging
+test-packaging:
+	@echo "Testing package image contents"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux1/x86_64" || echo "Missing amazonlinux1/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/aarch64" || echo "Missing amazonlinux2/aarch64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/amazonlinux2/x86_64" || echo "Missing amazonlinux2/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/ppc64le" || echo "Missing centos7/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos7/x86_64" || echo "Missing centos7/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/aarch64" || echo "Missing centos8/aarch64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/ppc64le" || echo "Missing centos8/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/centos8/x86_64" || echo "Missing centos8/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian10/amd64" || echo "Missing debian10/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/debian9/amd64" || echo "Missing debian9/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/opensuse-leap15.1/x86_64" || echo "Missing opensuse-leap15.1/x86_64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/amd64" || echo "Missing ubuntu16.04/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu16.04/ppc64le" || echo "Missing ubuntu16.04/ppc64le"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/amd64" || echo "Missing ubuntu18.04/amd64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/arm64" || echo "Missing ubuntu18.04/arm64"
+	@$(DOCKER) run --rm $(IMAGE) test -d "/artifacts/packages/ubuntu18.04/ppc64le" || echo "Missing ubuntu18.04/ppc64le"

From 738d28dac5479c1d5305e5965337b73b22887bda Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Thu, 25 Nov 2021 20:04:59 +0100
Subject: [PATCH 5/8] Add script to pull packages from packaging image

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 scripts/pull-packages.sh | 60 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)
 create mode 100755 scripts/pull-packages.sh

diff --git a/scripts/pull-packages.sh b/scripts/pull-packages.sh
new file mode 100755
index 00000000..b59fabb5
--- /dev/null
+++ b/scripts/pull-packages.sh
@@ -0,0 +1,60 @@
+#!/bin/bash
+
+# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+function assert_usage() {
+    echo "Incorrect arguments: $*"
+    echo "$(basename ${BASH_SOURCE[0]}) ROOT PACKAGE_NAME BUILD_VERSION"
+    exit 1
+}
+
+set -e -x
+
+SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
+PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
+
+if [[ $# -ne 0 ]]; then
+    assert_usage $*
+fi
+
+function pull_package_files() {
+    local image=$1
+    local dist_dir=$2
+    echo "Copying package files from ${image} to ${dist_dir}"
+    mkdir -p ${dist_dir}
+    docker run \
+        --rm \
+        --entrypoint="bash" \
+        -v $(pwd):$(pwd) \
+        -w $(pwd) \
+        -u $(id -u):$(id -g) \
+            ${image} \
+            -c "cp -R /artifacts/packages/* ${dist_dir}"
+}
+
+if [[ -z ${VERSION} ]]; then
+eval $(${SCRIPTS_DIR}/get-component-versions.sh)
+VERSION=${NVIDIA_CONTAINER_TOOLKIT_VERSION}${NVIDIA_CONTAINER_TOOLKIT_TAG:+-${NVIDIA_CONTAINER_TOOLKIT_TAG}}
+fi
+
+if [[ -z ${IMAGE} ]]; then
+: ${REGISTRY:=""}
+: ${IMAGE_NAME:="nvidia/container-toolkit"}
+image_tag=${VERSION}-packaging
+IMAGE=${REGISTRY:+${REGISTRY}/}${IMAGE_NAME}:${image_tag}
+fi
+: ${DIST_DIR:="dist-test"}
+
+pull_package_files ${IMAGE} ${DIST_DIR}

From fd5a1a72f055c54410212e27d0ae461f622ab58c Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Fri, 26 Nov 2021 15:15:46 +0100
Subject: [PATCH 6/8] Address review comments

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 build/container/Dockerfile.centos    | 11 +++---
 build/container/Dockerfile.packaging |  4 +--
 build/container/Dockerfile.ubuntu    | 11 +++---
 build/container/Makefile             | 48 +++++++++++++------------
 scripts/pull-packages.sh             | 52 +++++++++++++---------------
 5 files changed, 64 insertions(+), 62 deletions(-)

diff --git a/build/container/Dockerfile.centos b/build/container/Dockerfile.centos
index 35e65968..26b6bf95 100644
--- a/build/container/Dockerfile.centos
+++ b/build/container/Dockerfile.centos
@@ -46,17 +46,18 @@ ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
-ARG ARTIFACTS_DIR
-COPY ${ARTIFACTS_DIR}/ /artifacts/packages/
+ARG ARTIFACTS_ROOT
+ARG PACKAGE_DIST
+COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
 
 WORKDIR /artifacts/packages
 
 ARG PACKAGE_VERSION
 ARG PACKAGE_ARCH
 RUN yum localinstall -y \
-    ${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
-    ${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-${PACKAGE_VERSION}*.rpm \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit-${PACKAGE_VERSION}*.rpm
 
 WORKDIR /work
 
diff --git a/build/container/Dockerfile.packaging b/build/container/Dockerfile.packaging
index 444109c9..baeee6b4 100644
--- a/build/container/Dockerfile.packaging
+++ b/build/container/Dockerfile.packaging
@@ -21,8 +21,8 @@ FROM nvidia/cuda:${CUDA_VERSION}-base-${BASE_DIST}
 
 ENV NVIDIA_CONTAINER_TOOLKIT_VERSION="${VERSION}"
 
-ARG ARTIFACTS_DIR
-COPY ${ARTIFACTS_DIR}/ /artifacts/packages/
+ARG ARTIFACTS_ROOT
+COPY ${ARTIFACTS_ROOT} /artifacts/packages/
 
 WORKDIR /artifacts/packages
 
diff --git a/build/container/Dockerfile.ubuntu b/build/container/Dockerfile.ubuntu
index a1d610d9..ff3d30bd 100644
--- a/build/container/Dockerfile.ubuntu
+++ b/build/container/Dockerfile.ubuntu
@@ -52,17 +52,18 @@ ENV NVIDIA_DISABLE_REQUIRE="true"
 ENV NVIDIA_VISIBLE_DEVICES=all
 ENV NVIDIA_DRIVER_CAPABILITIES=utility
 
-ARG ARTIFACTS_DIR
-COPY ${ARTIFACTS_DIR}/ /artifacts/packages/
+ARG ARTIFACTS_ROOT
+ARG PACKAGE_DIST
+COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
 
 WORKDIR /artifacts/packages
 
 ARG PACKAGE_VERSION
 ARG PACKAGE_ARCH
 RUN dpkg -i \
-    ${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
-    ${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_${PACKAGE_VERSION}*.deb \
+    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit_${PACKAGE_VERSION}*.deb
 
 WORKDIR /work
 
diff --git a/build/container/Makefile b/build/container/Makefile
index 20893a49..c62e5b58 100644
--- a/build/container/Makefile
+++ b/build/container/Makefile
@@ -31,11 +31,13 @@ IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
 
 ##### Public rules #####
 DEFAULT_PUSH_TARGET := ubuntu18.04
-TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8 packaging
+TARGETS := ubuntu20.04 ubuntu18.04 ubi8 centos7 centos8
 
-BUILD_TARGETS := $(patsubst %, build-%, $(TARGETS))
-PUSH_TARGETS := $(patsubst %, push-%, $(TARGETS))
-TEST_TARGETS := $(patsubst %, test-%, $(filter-out packaging,$(TARGETS)))
+META_TARGETS := packaging
+
+BUILD_TARGETS := $(patsubst %,build-%,$(TARGETS) $(META_TARGETS))
+PUSH_TARGETS := $(patsubst %,push-%,$(TARGETS) $(META_TARGETS))
+TEST_TARGETS := $(patsubst %,test-%, $(TARGETS))
 
 .PHONY: $(TARGETS) $(PUSH_TARGETS) $(BUILD_TARGETS) $(TEST_TARGETS)
 
@@ -57,15 +59,19 @@ push-short:
 build-%: DIST = $(*)
 build-%: DOCKERFILE = $(CURDIR)/build/container/Dockerfile.$(DOCKERFILE_SUFFIX)
 
+ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
+
 # Use a generic build target to build the relevant images
-$(BUILD_TARGETS): build-%: $(ARTIFACTS_DIR)
+$(BUILD_TARGETS): build-%: $(ARTIFACTS_ROOT)
 	DOCKER_BUILDKIT=1 \
-		$(DOCKER) build --platform=linux/amd64 --pull \
+		$(DOCKER) build --pull \
+		--platform=linux/amd64 \
 		--tag $(IMAGE) \
-		--build-arg ARTIFACTS_DIR="$(ARTIFACTS_DIR)" \
+		--build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \
 		--build-arg BASE_DIST="$(BASE_DIST)" \
 		--build-arg CUDA_VERSION="$(CUDA_VERSION)" \
 		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
+		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg PACKAGE_ARCH="$(PACKAGE_ARCH)" \
 		--build-arg VERSION="$(VERSION)" \
@@ -73,34 +79,30 @@ $(BUILD_TARGETS): build-%: $(ARTIFACTS_DIR)
 		$(CURDIR)
 
 
-ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
-ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/$(*)
-
-PACKAGE_ARCH = amd64
-
+build-ubuntu%: BASE_DIST = $(*)
 build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
+build-ubuntu%: PACKAGE_ARCH := amd64
+build-ubuntu%: PACKAGE_DIST = $(BASE_DIST)
 build-ubuntu%: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),~$(LIB_TAG))
 
-build-ubuntu18.04: BASE_DIST := ubuntu18.04
-build-ubuntu20.04: BASE_DIST := ubuntu20.04
-
-build-ubi8: DOCKERFILE_SUFFIX := centos
 # TODO: Update this to use the centos8 packages
-build-ubi8: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)/centos7
-build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 build-ubi8: BASE_DIST := ubi8
+build-ubi8: DOCKERFILE_SUFFIX := centos
 build-ubi8: PACKAGE_ARCH := x86_64
+build-ubi8: PACKAGE_DIST = centos7
+build-ubi8: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
+build-centos%: BASE_DIST = $(*)
 build-centos%: DOCKERFILE_SUFFIX := centos
-build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 build-centos%: PACKAGE_ARCH := x86_64
-
-build-centos7: BASE_DIST := centos7
-build-centos8: BASE_DIST := centos8
+build-centos%: PACKAGE_DIST = $(BASE_DIST)
+build-centos%: PACKAGE_VERSION := $(LIB_VERSION)-$(if $(LIB_TAG),0.1.$(LIB_TAG),1)
 
 build-packaging: BASE_DIST := ubuntu20.04
 build-packaging: DOCKERFILE_SUFFIX := packaging
-build-packaging: ARTIFACTS_DIR = $(ARTIFACTS_ROOT)
+build-packaging: PACKAGE_ARCH := amd64
+build-packaging: PACKAGE_DIST = all
+build-packaging: PACKAGE_VERSION := $(LIB_VERSION)$(if $(LIB_TAG),-$(LIB_TAG))
 
 # Test targets
 test-%: DIST = $(*)
diff --git a/scripts/pull-packages.sh b/scripts/pull-packages.sh
index b59fabb5..f62e0bb4 100755
--- a/scripts/pull-packages.sh
+++ b/scripts/pull-packages.sh
@@ -16,7 +16,7 @@
 
 function assert_usage() {
     echo "Incorrect arguments: $*"
-    echo "$(basename ${BASH_SOURCE[0]}) ROOT PACKAGE_NAME BUILD_VERSION"
+    echo "$(basename ${BASH_SOURCE[0]}) IMAGE DIST_DIR"
     exit 1
 }
 
@@ -25,36 +25,34 @@ set -e -x
 SCRIPTS_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../scripts && pwd )"
 PROJECT_ROOT="$( cd ${SCRIPTS_DIR}/.. && pwd )"
 
-if [[ $# -ne 0 ]]; then
+if [[ $# -ne 2 ]]; then
     assert_usage $*
 fi
 
-function pull_package_files() {
-    local image=$1
-    local dist_dir=$2
-    echo "Copying package files from ${image} to ${dist_dir}"
-    mkdir -p ${dist_dir}
-    docker run \
-        --rm \
-        --entrypoint="bash" \
-        -v $(pwd):$(pwd) \
-        -w $(pwd) \
-        -u $(id -u):$(id -g) \
-            ${image} \
-            -c "cp -R /artifacts/packages/* ${dist_dir}"
-}
-
-if [[ -z ${VERSION} ]]; then
-eval $(${SCRIPTS_DIR}/get-component-versions.sh)
-VERSION=${NVIDIA_CONTAINER_TOOLKIT_VERSION}${NVIDIA_CONTAINER_TOOLKIT_TAG:+-${NVIDIA_CONTAINER_TOOLKIT_TAG}}
-fi
+IMAGE=$1
+DIST_DIR=$2
 
 if [[ -z ${IMAGE} ]]; then
-: ${REGISTRY:=""}
-: ${IMAGE_NAME:="nvidia/container-toolkit"}
-image_tag=${VERSION}-packaging
-IMAGE=${REGISTRY:+${REGISTRY}/}${IMAGE_NAME}:${image_tag}
+    echo "ERROR: IMAGE must be non-empty"
+    exit 1
 fi
-: ${DIST_DIR:="dist-test"}
 
-pull_package_files ${IMAGE} ${DIST_DIR}
+if [[ -z ${DIST_DIR} ]]; then
+    echo "ERROR: DIST_DIR must be non-empty"
+    exit 1
+fi
+
+if [[ -e ${DIST_DIR} ]]; then
+    echo "ERROR: The specified DIST_DIR ${DIST_DIR} exists."
+    exit 1
+fi
+
+echo "Copying package files from ${IMAGE} to ${DIST_DIR}"
+mkdir -p ${DIST_DIR}
+docker run --rm \
+    -v $(pwd):$(pwd) \
+    -w $(pwd) \
+    -u $(id -u):$(id -g) \
+    --entrypoint="bash" \
+        ${IMAGE} \
+        -c "cp -R /artifacts/packages/* ${DIST_DIR}"

From 2d07385e81011dbe0d24463035f602e3fc9329cc Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Fri, 26 Nov 2021 15:49:30 +0100
Subject: [PATCH 7/8] Pull public staging images to scan and release

This change pulls images from public staging repositories to scan
and release. This ensures that the bits built and tested in public
CI (off the master branch, for example) match those scanned and
released. This also serves to reduce the load on our internal CI
runners as these don't have to store artifacts and build images.

Two CI variables: STAGING_REGISTRY and STAGING_VERSION are used
to control which image is pulled for release, with the latter
defaulting to the CI_COMMIT_SHORT_SHA.

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 .common-ci.yml | 319 -------------------------------------------------
 .gitlab-ci.yml | 319 +++++++++++++++++++++++++++++++++++++++++++++++++
 .nvidia-ci.yml |  52 ++++++++
 3 files changed, 371 insertions(+), 319 deletions(-)

diff --git a/.common-ci.yml b/.common-ci.yml
index 3343bcc2..bfd50503 100644
--- a/.common-ci.yml
+++ b/.common-ci.yml
@@ -34,69 +34,6 @@ stages:
   - release
   - build-all
 
-build-dev-image:
-  stage: image
-  script:
-    - apk --no-cache add make bash
-    - make .build-image
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    - make .push-build-image
-
-.requires-build-image:
-  image: "${BUILDIMAGE}"
-
-.go-check:
-  extends:
-    - .requires-build-image
-  stage: go-checks
-
-fmt:
-  extends:
-    - .go-check
-  script:
-    - make assert-fmt
-
-vet:
-  extends:
-    - .go-check
-  script:
-    - make vet
-
-lint:
-  extends:
-    - .go-check
-  script:
-    - make lint
-  allow_failure: true
-
-ineffassign:
-  extends:
-    - .go-check
-  script:
-    - make ineffassign
-  allow_failure: true
-
-misspell:
-  extends:
-    - .go-check
-  script:
-    - make misspell
-
-go-build:
-  extends:
-    - .requires-build-image
-  stage: go-build
-  script:
-    - make build
-
-unit-tests:
-  extends:
-    - .requires-build-image
-  stage: unit-tests
-  script:
-    - make coverage
-
-
 # Define the distribution targets
 .dist-amazonlinux1:
   variables:
@@ -163,206 +100,6 @@ unit-tests:
   variables:
     ARCH: x86_64
 
-# Define the package build helpers
-.multi-arch-build:
-  before_script:
-    - apk add --no-cache coreutils build-base sed git bash make
-    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
-
-.package-artifacts:
-  variables:
-    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
-    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
-    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
-
-.package-build:
-  extends:
-    - .multi-arch-build
-    - .package-artifacts
-  stage: package-build
-  script:
-    - ./scripts/release.sh ${DIST}-${ARCH}
-
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_ROOT}
-
-# Define the package build targets
-package-amazonlinux1-x86_64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux1
-    - .arch-x86_64
-
-package-amazonlinux2-aarch64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-aarch64
-
-package-amazonlinux2-x86_64:
-  extends:
-    - .package-build
-    - .dist-amazonlinux2
-    - .arch-x86_64
-
-package-centos7-ppc64le:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-ppc64le
-
-package-centos7-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-x86_64
-
-package-centos8-aarch64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-aarch64
-
-package-centos8-ppc64le:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-ppc64le
-
-package-centos8-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-x86_64
-
-package-debian10-amd64:
-  extends:
-    - .package-build
-    - .dist-debian10
-    - .arch-amd64
-
-package-debian9-amd64:
-  extends:
-    - .package-build
-    - .dist-debian9
-    - .arch-amd64
-
-package-opensuse-leap15.1-x86_64:
-  extends:
-    - .package-build
-    - .dist-opensuse-leap15.1
-    - .arch-x86_64
-
-package-ubuntu16.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-amd64
-
-package-ubuntu16.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu16.04
-    - .arch-ppc64le
-
-package-ubuntu18.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-amd64
-
-package-ubuntu18.04-arm64:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-arm64
-
-package-ubuntu18.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-ppc64le
-
-# Define the image build targets
-.image-build:
-  stage: image-build
-  variables:
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - apk add --no-cache bash make
-    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-  script:
-    - make -f build/container/Makefile build-${DIST}
-    - make -f build/container/Makefile push-${DIST}
-
-image-centos7:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-centos7
-  needs:
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
-
-image-centos8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-centos8
-  needs:
-    - package-centos8-aarch64
-    - package-centos8-x86_64
-    - package-centos8-ppc64le
-
-image-ubi8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubi8
-  needs:
-    # Note: The ubi8 image currently uses the centos7 packages
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
-
-image-ubuntu18.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu18.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
-
-# The DIST=packaging target creates an image containing all built packages
-image-packaging:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-packaging
-  needs:
-    - package-amazonlinux1-x86_64
-    - package-amazonlinux2-aarch64
-    - package-amazonlinux2-x86_64
-    - package-centos7-ppc64le
-    - package-centos7-x86_64
-    - package-centos8-aarch64
-    - package-centos8-ppc64le
-    - package-centos8-x86_64
-    - package-debian10-amd64
-    - package-debian9-amd64
-    - package-opensuse-leap15.1-x86_64
-    - package-ubuntu16.04-amd64
-    - package-ubuntu16.04-ppc64le
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - package-ubuntu18.04-ppc64le
-
-
 # Define test helpers
 .integration:
   stage: test
@@ -376,63 +113,7 @@ image-packaging:
   script:
     - make -f build/container/Makefile test-${DIST}
 
-.test:toolkit:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "toolkit"
-
-.test:docker:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "docker"
-
-.test:containerd:
-  # TODO: The containerd tests fail due to issues with SIGHUP.
-  # Until this is resolved with retry up to twice and allow failure here.
-  retry: 2
-  allow_failure: true
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "containerd"
-
-.test:crio:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "crio"
-
 # Define the test targets
-test-toolkit-ubuntu18.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-containerd-ubuntu18.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-crio-ubuntu18.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
-test-docker-ubuntu18.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu18.04
-  needs:
-    - image-ubuntu18.04
-
 test-packaging:
   extends:
     - .integration
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 81d2abfb..f716cae0 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -15,6 +15,325 @@
 include:
   - .common-ci.yml
 
+build-dev-image:
+  stage: image
+  script:
+    - apk --no-cache add make bash
+    - make .build-image
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+    - make .push-build-image
+
+.requires-build-image:
+  image: "${BUILDIMAGE}"
+
+.go-check:
+  extends:
+    - .requires-build-image
+  stage: go-checks
+
+fmt:
+  extends:
+    - .go-check
+  script:
+    - make assert-fmt
+
+vet:
+  extends:
+    - .go-check
+  script:
+    - make vet
+
+lint:
+  extends:
+    - .go-check
+  script:
+    - make lint
+  allow_failure: true
+
+ineffassign:
+  extends:
+    - .go-check
+  script:
+    - make ineffassign
+  allow_failure: true
+
+misspell:
+  extends:
+    - .go-check
+  script:
+    - make misspell
+
+go-build:
+  extends:
+    - .requires-build-image
+  stage: go-build
+  script:
+    - make build
+
+unit-tests:
+  extends:
+    - .requires-build-image
+  stage: unit-tests
+  script:
+    - make coverage
+
+# Define the package build helpers
+.multi-arch-build:
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
+
+.package-artifacts:
+  variables:
+    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
+    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
+    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
+
+.package-build:
+  extends:
+    - .multi-arch-build
+    - .package-artifacts
+  stage: package-build
+  script:
+    - ./scripts/release.sh ${DIST}-${ARCH}
+
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
+
+# Define the package build targets
+package-amazonlinux1-x86_64:
+  extends:
+    - .package-build
+    - .dist-amazonlinux1
+    - .arch-x86_64
+
+package-amazonlinux2-aarch64:
+  extends:
+    - .package-build
+    - .dist-amazonlinux2
+    - .arch-aarch64
+
+package-amazonlinux2-x86_64:
+  extends:
+    - .package-build
+    - .dist-amazonlinux2
+    - .arch-x86_64
+
+package-centos7-ppc64le:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-ppc64le
+
+package-centos7-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos7
+    - .arch-x86_64
+
+package-centos8-aarch64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-aarch64
+
+package-centos8-ppc64le:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-ppc64le
+
+package-centos8-x86_64:
+  extends:
+    - .package-build
+    - .dist-centos8
+    - .arch-x86_64
+
+package-debian10-amd64:
+  extends:
+    - .package-build
+    - .dist-debian10
+    - .arch-amd64
+
+package-debian9-amd64:
+  extends:
+    - .package-build
+    - .dist-debian9
+    - .arch-amd64
+
+package-opensuse-leap15.1-x86_64:
+  extends:
+    - .package-build
+    - .dist-opensuse-leap15.1
+    - .arch-x86_64
+
+package-ubuntu16.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-amd64
+
+package-ubuntu16.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu16.04
+    - .arch-ppc64le
+
+package-ubuntu18.04-amd64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-amd64
+
+package-ubuntu18.04-arm64:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-arm64
+
+package-ubuntu18.04-ppc64le:
+  extends:
+    - .package-build
+    - .dist-ubuntu18.04
+    - .arch-ppc64le
+
+# Define the image build targets
+.image-build:
+  stage: image-build
+  variables:
+    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    VERSION: "${CI_COMMIT_SHORT_SHA}"
+  before_script:
+    - apk add --no-cache bash make
+    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
+    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+  script:
+    - make -f build/container/Makefile build-${DIST}
+    - make -f build/container/Makefile push-${DIST}
+
+image-centos7:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos7
+  needs:
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
+
+image-centos8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-centos8
+  needs:
+    - package-centos8-aarch64
+    - package-centos8-x86_64
+    - package-centos8-ppc64le
+
+image-ubi8:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubi8
+  needs:
+    # Note: The ubi8 image currently uses the centos7 packages
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
+
+image-ubuntu18.04:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-ubuntu18.04
+  needs:
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
+
+# The DIST=packaging target creates an image containing all built packages
+image-packaging:
+  extends:
+    - .image-build
+    - .package-artifacts
+    - .dist-packaging
+  needs:
+    - package-amazonlinux1-x86_64
+    - package-amazonlinux2-aarch64
+    - package-amazonlinux2-x86_64
+    - package-centos7-ppc64le
+    - package-centos7-x86_64
+    - package-centos8-aarch64
+    - package-centos8-ppc64le
+    - package-centos8-x86_64
+    - package-debian10-amd64
+    - package-debian9-amd64
+    - package-opensuse-leap15.1-x86_64
+    - package-ubuntu16.04-amd64
+    - package-ubuntu16.04-ppc64le
+    - package-ubuntu18.04-amd64
+    - package-ubuntu18.04-arm64
+    - package-ubuntu18.04-ppc64le
+
+# Define publish test helpers
+.test:toolkit:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "toolkit"
+
+.test:docker:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "docker"
+
+.test:containerd:
+  # TODO: The containerd tests fail due to issues with SIGHUP.
+  # Until this is resolved with retry up to twice and allow failure here.
+  retry: 2
+  allow_failure: true
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "containerd"
+
+.test:crio:
+  extends:
+    - .integration
+  variables:
+    TEST_CASES: "crio"
+
+# Define the test targets
+test-toolkit-ubuntu18.04:
+  extends:
+    - .test:toolkit
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-containerd-ubuntu18.04:
+  extends:
+    - .test:containerd
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-crio-ubuntu18.04:
+  extends:
+    - .test:crio
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
+test-docker-ubuntu18.04:
+  extends:
+    - .test:docker
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
 # build-all jobs build packages for every OS / ARCH combination we support.
 #
 # They are run under two conditions:
diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml
index 83ed0037..9f34aa7d 100644
--- a/.nvidia-ci.yml
+++ b/.nvidia-ci.yml
@@ -32,6 +32,51 @@ variables:
   DEVEL_RELEASE_IMAGE_VERSION: "devel"
   # On the multi-arch builder we don't need the qemu setup.
   SKIP_QEMU_SETUP: "1"
+  # Define the public staging registry
+  STAGING_REGISTRY: registry.gitlab.com/nvidia/container-toolkit/container-toolkit/staging
+  STAGING_VERSION: ${CI_COMMIT_SHORT_SHA}
+
+.image-pull:
+  stage: image-build
+  variables:
+    IN_REGISTRY: "${STAGING_REGISTRY}"
+    IN_IMAGE_NAME: container-toolkit
+    IN_VERSION: "${STAGING_VERSION}"
+    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    OUT_REGISTRY: "${CI_REGISTRY}"
+    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+  script:
+    - docker pull ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}
+    - docker tag ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} ${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST}
+    - docker login -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" "${OUT_REGISTRY}"
+    - docker push ${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST}
+
+image-centos7:
+  extends:
+    - .image-pull
+    - .dist-centos7
+
+image-centos8:
+  extends:
+    - .image-pull
+    - .dist-centos8
+
+image-ubi8:
+  extends:
+    - .image-pull
+    - .dist-ubi8
+
+image-ubuntu18.04:
+  extends:
+    - .image-pull
+    - .dist-ubuntu18.04
+
+# The DIST=packaging target creates an image containing all built packages
+image-packaging:
+  extends:
+    - .image-pull
+    - .dist-packaging
 
 # We skip the integration tests for the internal CI:
 .integration:
@@ -130,6 +175,13 @@ scan-ubi8:
     # TODO: For now we disable external releases
     DOCKER: echo
 
+release:staging-ubuntu18.04:
+  extends:
+    - .release:staging
+    - .dist-ubuntu18.04
+  needs:
+    - image-ubuntu18.04
+
 # Define the external release targets
 # Release to NGC
 release:ngc-centos7:

From b334f1977b6826fefeb0a11f7576af36ae09e80d Mon Sep 17 00:00:00 2001
From: Evan Lezar <elezar@nvidia.com>
Date: Thu, 2 Dec 2021 17:52:44 +0100
Subject: [PATCH 8/8] Add delay and timeout to image pull job

Signed-off-by: Evan Lezar <elezar@nvidia.com>
---
 .nvidia-ci.yml | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/.nvidia-ci.yml b/.nvidia-ci.yml
index 9f34aa7d..09c9a6f0 100644
--- a/.nvidia-ci.yml
+++ b/.nvidia-ci.yml
@@ -46,6 +46,18 @@ variables:
     OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
     OUT_REGISTRY: "${CI_REGISTRY}"
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+  # We delay the job start to allow the public pipeline to generate the required images.
+  when: delayed
+  start_in: 30 minutes
+  timeout: 30 minutes
+  retry:
+    max: 2
+    when:
+      - job_execution_timeout
+      - stuck_or_timeout_failure
+  before_script:
+    - >
+      docker pull ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
   script:
     - docker pull ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}
     - docker tag ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} ${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST}