Merge branch 'nvidia-docker-as-meta-package' into 'main'

Support building nvidia-docker and nvidia-container-runtime as dist-independent packages

See merge request nvidia/container-toolkit/container-toolkit!379

.gitlab-ci.yml

@@ -102,8 +102,30 @@ unit-tests:
     name: ${ARTIFACTS_NAME}
     paths:
       - ${ARTIFACTS_ROOT}
+  needs:
+    - job: package-meta-packages
+      artifacts: true
 
 # Define the package build targets
+package-meta-packages:
+  extends:
+    - .package-artifacts
+  stage: package-build
+  variables:
+    SKIP_LIBNVIDIA_CONTAINER: "yes"
+    SKIP_NVIDIA_CONTAINER_TOOLKIT: "yes"
+  parallel:
+    matrix:
+      - PACKAGING: [deb, rpm]
+  before_script:
+    - apk add --no-cache coreutils build-base sed git bash make
+  script:
+    - ./scripts/build-packages.sh ${PACKAGING}
+  artifacts:
+    name: ${ARTIFACTS_NAME}
+    paths:
+      - ${ARTIFACTS_ROOT}
+
 package-amazonlinux2-aarch64:
   extends:
     - .package-build

scripts/build-all-components.sh

@@ -36,6 +36,8 @@ fi
 
 TARGET=$1
 
+source "${SCRIPTS_DIR}"/utils.sh
+
 : "${DIST_DIR:=${PROJECT_ROOT}/dist}"
 export DIST_DIR
 
@@ -49,42 +51,72 @@ echo "Building ${TARGET} for all packages to ${DIST_DIR}"
 
 "${SCRIPTS_DIR}/get-component-versions.sh"
 
-# Build libnvidia-container
-make -C "${LIBNVIDIA_CONTAINER_ROOT}" -f mk/docker.mk "${TARGET}"
-
 if [[ -z "${NVIDIA_CONTAINER_TOOLKIT_VERSION}" || -z "${LIBNVIDIA_CONTAINER_VERSION}" ]]; then
 eval $(${SCRIPTS_DIR}/get-component-versions.sh)
 fi
 
+# Build libnvidia-container
+if [[ -z ${SKIP_LIBNVIDIA_CONTAINER} ]]; then
+    make -C "${LIBNVIDIA_CONTAINER_ROOT}" -f mk/docker.mk "${TARGET}"
+fi
+
+if [[ -z ${SKIP_NVIDIA_CONTAINER_TOOLKIT} ]]; then
 # Build nvidia-container-toolkit
 make -C "${NVIDIA_CONTAINER_TOOLKIT_ROOT}" \
     LIBNVIDIA_CONTAINER_VERSION="${LIBNVIDIA_CONTAINER_VERSION}" \
     LIBNVIDIA_CONTAINER_TAG="${LIBNVIDIA_CONTAINER_TAG}" \
         "${TARGET}"
+fi
 
 # If required we also build the nvidia-container-runtime and nvidia-docker packages.
 # Since these are essentially meta packages intended to allow for users to
 # transition from older installation workflows, we skip these for rc builds
 # (NVIDIA_CONTAINER_TOOLKIT_TAG != "") and releases with a non-zero patch
 # version of 0.
-if [[ -z ${NVIDIA_CONTAINER_TOOLKIT_TAG} && "${NVIDIA_CONTAINER_TOOLKIT_VERSION%.0}" != "${NVIDIA_CONTAINER_TOOLKIT_VERSION}" ]]; then
+if [[ -n ${FORCE_META_PACKAGES} || -z ${NVIDIA_CONTAINER_TOOLKIT_TAG} && "${NVIDIA_CONTAINER_TOOLKIT_VERSION%.0}" != "${NVIDIA_CONTAINER_TOOLKIT_VERSION}" ]]; then
+    package_format=$(package_type ${TARGET})
+    package_target=$(get_package_target ${TARGET})
+
     # We set the TOOLKIT_VERSION, TOOLKIT_TAG for the nvidia-container-runtime and nvidia-docker targets
     # The LIB_TAG is also overridden to match the TOOLKIT_TAG.
-    # Build nvidia-container-runtime
-    make -C ${NVIDIA_CONTAINER_RUNTIME_ROOT} \
-        LIB_VERSION="${NVIDIA_CONTAINER_RUNTIME_VERSION}" \
-        LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
-        TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
-        TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
-            ${TARGET}
 
-    # Build nvidia-docker2
-    make -C ${NVIDIA_DOCKER_ROOT} \
-        LIB_VERSION="${NVIDIA_DOCKER_VERSION}" \
-        LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
-        TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
-        TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
-            ${TARGET}
+    # Build nvidia-container-runtime if required
+    package_name="nvidia-container-runtime"
+    package_version=${NVIDIA_CONTAINER_RUNTIME_VERSION}${NVIDIA_CONTAINER_TOOLKIT_TAG:+~${NVIDIA_CONTAINER_TOOLKIT_TAG}}-1
+    package_pattern=${DIST_DIR}/${package_format}/all/${package_name}?${package_version}?*.${package_format}
+    package=$(ls ${package_pattern}) || echo ""
+    if [[ -z ${package} ]]; then
+        echo "${package_name} does not exist"
+        make -C ${NVIDIA_CONTAINER_RUNTIME_ROOT} \
+            LIB_VERSION="${NVIDIA_CONTAINER_RUNTIME_VERSION}" \
+            LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+            TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
+            TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+                ${TARGET}
+    fi
+    if [[ -n ${package_target} ]]; then
+        mkdir -p ${DIST_DIR}/${package_target}/
+        cp -p ${package_pattern} ${DIST_DIR}/${package_target}/
+    fi
+
+    # Build nvidia-docker2 if required
+    package_name="nvidia-docker2"
+    package_version=${NVIDIA_DOCKER_VERSION}${NVIDIA_CONTAINER_TOOLKIT_TAG:+~${NVIDIA_CONTAINER_TOOLKIT_TAG}}-1
+    package_pattern=${DIST_DIR}/${package_format}/all/${package_name}?${package_version}?*.${package_format}
+    package=$(ls ${package_pattern}) || echo ""
+    if [[ -z ${package} ]]; then
+        echo "${package_name} does not exist"
+        make -C ${NVIDIA_DOCKER_ROOT} \
+            LIB_VERSION="${NVIDIA_DOCKER_VERSION}" \
+            LIB_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+            TOOLKIT_VERSION="${NVIDIA_CONTAINER_TOOLKIT_VERSION}" \
+            TOOLKIT_TAG="${NVIDIA_CONTAINER_TOOLKIT_TAG}" \
+                ${TARGET}
+    fi
+    if [[ -n ${package_target} ]]; then
+        mkdir -p ${DIST_DIR}/${package_target}/
+        cp -p ${package_pattern} ${DIST_DIR}/${package_target}/
+    fi
 
 else
     echo "Skipping nvidia-container-runtime and nvidia-docker builds."

scripts/build-packages.sh

@@ -46,12 +46,16 @@ else
     targets=${all[@]}
 fi
 
-echo "Updating components"
-"${SCRIPTS_DIR}/update-components.sh"
-if [[ -n $(git status -s third_party) && ${ALLOW_LOCAL_COMPONENT_CHANGES} != "true" ]]; then
-    echo "ERROR: Building with local component changes."
-    echo "Commit pending changes or rerun with ALLOW_LOCAL_COMPONENT_CHANGES='true'"
-    exit 1
+if [[ -z ${SKIP_UPDATE_COMPONENTS} ]]; then
+    echo "Updating components"
+    "${SCRIPTS_DIR}/update-components.sh"
+    if [[ -n $(git status -s third_party) && ${ALLOW_LOCAL_COMPONENT_CHANGES} != "true" ]]; then
+        echo "ERROR: Building with local component changes."
+        echo "Commit pending changes or rerun with ALLOW_LOCAL_COMPONENT_CHANGES='true'"
+        exit 1
+    fi
+else
+    echo "Skipping update of components"
 fi
 
 eval $(${SCRIPTS_DIR}/get-component-versions.sh)

scripts/utils.sh

@@ -16,6 +16,10 @@ function package_type() {
         ;;
     ubuntu*) pkg_type=deb
         ;;
+    deb) pkg_type=deb
+        ;;
+    rpm) pkg_type=rpm
+        ;;
     *) exit 1
         ;;
     esac
@@ -34,3 +38,18 @@ function get_artifactory_repository() {
         ;;
     esac
 }
+
+function get_package_target() {
+    local target=$1
+    local dist=${target%-*}
+    local arch=${target##*-}
+
+    case ${target} in
+    deb) echo ""
+        ;;
+    rpm) echo ""
+        ;;
+    *) echo "${dist}/${arch}"
+        ;;
+    esac
+}

third_party/libnvidia-container

@@ -1 +1 @@
-Subproject commit bbb74b9675de980c61d1f9c8f339efa8cc53f5ad
+Subproject commit eb0415c458c5e5d97cb8ac08b42803d075ed73cd

third_party/nvidia-container-runtime

@@ -1 +1 @@
-Subproject commit 68b81a207bc936328955621708862ec72c6225d4
+Subproject commit 6f328ae638853b5b295deabfa08becc74f7273ef

third_party/nvidia-docker

@@ -1 +1 @@
-Subproject commit 80902fe3afab0b08502a47b2f0e134869f813aa2
+Subproject commit 8bef6b0a49951bfde3426ae80dba5e5810b15ee8