Extract device information from host path if possible

This change extracts the required information from the host device node if available. Signed-off-by: Evan Lezar <elezar@nvidia.com>
Merge pull request #1153 from elezar/switch-to-ubi9
2025-06-26 18:18:24 +00:00 · 2025-06-19 15:20:31 +02:00 · 2025-06-18 13:10:32 +02:00 · 2025-06-18 11:58:23 +02:00 · 2025-06-18 11:56:14 +02:00 · 2025-06-18 11:51:52 +02:00
32 changed files with 1101 additions and 930 deletions
--- a/.common-ci.yml
+++ b/.common-ci.yml
@@ -22,15 +22,7 @@ variables:
  BUILD_MULTI_ARCH_IMAGES: "true"

 stages:
-  - trigger
-  - image
-  - lint
-  - go-checks
-  - go-build
-  - unit-tests
-  - package-build
-  - image-build
-  - test
+  - pull
  - scan
  - release
  - sign
@@ -53,108 +45,6 @@ workflow:
    # We then add all the regular triggers
    - !reference [.pipeline-trigger-rules, rules]

-# The main or manual job is used to filter out distributions or architectures that are not required on
-# every build.
-.main-or-manual:
-  rules:
-    - !reference [.pipeline-trigger-rules, rules]
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: manual
-
-# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
-trigger-pipeline:
-  stage: trigger
-  script:
-    - echo "starting pipeline"
-  rules:
-    - !reference [.main-or-manual, rules]
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      allow_failure: false
-    - when: always
-
-# Define the distribution targets
-.dist-centos7:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: centos7
-
-.dist-centos8:
-  variables:
-    DIST: centos8
-
-.dist-ubi8:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: ubi8
-
-.dist-ubuntu18.04:
-  variables:
-    DIST: ubuntu18.04
-
-.dist-ubuntu20.04:
-  variables:
-    DIST: ubuntu20.04
-
-.dist-packaging:
-  variables:
-    DIST: packaging
-
-# Define architecture targets
-.arch-aarch64:
-  variables:
-    ARCH: aarch64
-
-.arch-amd64:
-  variables:
-    ARCH: amd64
-
-.arch-arm64:
-  variables:
-    ARCH: arm64
-
-.arch-ppc64le:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    ARCH: ppc64le
-
-.arch-x86_64:
-  variables:
-    ARCH: x86_64
-
-# Define the platform targets
-.platform-amd64:
-  variables:
-    PLATFORM: linux/amd64
-
-.platform-arm64:
-  variables:
-    PLATFORM: linux/arm64
-
-# Define test helpers
-.integration:
-  stage: test
-  variables:
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - apk add --no-cache make bash jq
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
-  script:
-    - make -f deployments/container/Makefile test-${DIST}
-
-# Define the test targets
-test-packaging:
-  extends:
-    - .integration
-    - .dist-packaging
-  needs:
-    - image-packaging
-
 # Download the regctl binary for use in the release steps
 .regctl-setup:
  before_script:
@@ -164,84 +54,3 @@ test-packaging:
    - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
    - chmod a+x bin/regctl
    - export PATH=$(pwd)/bin:${PATH}
-
-# .release forms the base of the deployment jobs which push images to the CI registry.
-# This is extended with the version to be deployed (e.g. the SHA or TAG) and the
-# target os.
-.release:
-  stage: release
-  variables:
-    # Define the source image for the release
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-    # OUT_IMAGE_VERSION is overridden for external releases
-    OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - !reference [.regctl-setup, before_script]
-    # We ensure that the components of the output image are set:
-    - 'echo Image Name: ${OUT_IMAGE_NAME} ; [[ -n "${OUT_IMAGE_NAME}" ]] || exit 1'
-    - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
-
-    - apk add --no-cache make bash
-  script:
-    # Log in to the "output" registry, tag the image and push the image
-    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
-    - regctl registry login "${CI_REGISTRY}" -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}"
-    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || echo "Logging in to output registry ${OUT_REGISTRY}"'
-    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"'
-
-    # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the
-    # Target
-    - make -f deployments/container/Makefile push-${DIST}
-
-# Define a staging release step that pushes an image to an internal "staging" repository
-# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
-# outside of the release process.
-.release:staging:
-  extends:
-    - .release
-  variables:
-    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
-    OUT_REGISTRY: "${NGC_REGISTRY}"
-    OUT_IMAGE_NAME: "${NGC_REGISTRY_STAGING_IMAGE_NAME}"
-
-# Define an external release step that pushes an image to an external repository.
-# This includes a devlopment image off main.
-.release:external:
-  extends:
-    - .release
-  variables:
-    FORCE_PUBLISH_IMAGES: "yes"
-  rules:
-    - if: $CI_COMMIT_TAG
-      variables:
-        OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}"
-    - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH
-      variables:
-        OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
-
-# Define the release jobs
-release:staging-ubi8:
-  extends:
-    - .release:staging
-    - .dist-ubi8
-  needs:
-    - image-ubi8
-
-release:staging-ubuntu20.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu20.04
-  needs:
-    - test-toolkit-ubuntu20.04
-    - test-containerd-ubuntu20.04
-    - test-crio-ubuntu20.04
-    - test-docker-ubuntu20.04
-
-release:staging-packaging:
-  extends:
-    - .release:staging
-    - .dist-packaging
-  needs:
-    - test-packaging
--- a/.github/workflows/e2e.yaml
+++ b/.github/workflows/e2e.yaml
@@ -72,7 +72,7 @@ jobs:
        env:
          E2E_INSTALL_CTK: "true"
          E2E_IMAGE_NAME: ghcr.io/nvidia/container-toolkit
-          E2E_IMAGE_TAG: ${{ inputs.version }}-ubuntu20.04
+          E2E_IMAGE_TAG: ${{ inputs.version }}
          E2E_SSH_USER: ${{ secrets.E2E_SSH_USER }}
          E2E_SSH_HOST: ${{ steps.holodeck_public_dns_name.outputs.result }}
        run: |
--- a/.github/workflows/image.yaml
+++ b/.github/workflows/image.yaml
@@ -80,14 +80,8 @@ jobs:
    strategy:
      matrix:
        dist:
-          - ubuntu20.04
-          - ubi8
+          - ubi9
          - packaging
-        ispr:
-          - ${{ github.ref_name != 'main' && !startsWith( github.ref_name, 'release-' ) }}
-        exclude:
-          - ispr: true
-            dist: ubi8
    needs: packages
    steps:
      - uses: actions/checkout@v4
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,228 +0,0 @@
-# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-include:
-  - .common-ci.yml
-
-# Define the package build helpers
-.multi-arch-build:
-  before_script:
-    - apk add --no-cache coreutils build-base sed git bash make
-    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
-
-.package-artifacts:
-  variables:
-    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
-    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
-    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
-
-.package-build:
-  extends:
-    - .multi-arch-build
-    - .package-artifacts
-  stage: package-build
-  timeout: 3h
-  script:
-    - ./scripts/build-packages.sh ${DIST}-${ARCH}
-
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_ROOT}
-  needs:
-    - job: package-meta-packages
-      artifacts: true
-
-# Define the package build targets
-package-meta-packages:
-  extends:
-    - .package-artifacts
-  stage: package-build
-  variables:
-    SKIP_LIBNVIDIA_CONTAINER: "yes"
-    SKIP_NVIDIA_CONTAINER_TOOLKIT: "yes"
-  parallel:
-    matrix:
-      - PACKAGING: [deb, rpm]
-  before_script:
-    - apk add --no-cache coreutils build-base sed git bash make
-  script:
-    - ./scripts/build-packages.sh ${PACKAGING}
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_ROOT}
-
-package-centos7-aarch64:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-aarch64
-
-package-centos7-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-x86_64
-
-package-centos8-ppc64le:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-ppc64le
-
-package-ubuntu18.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-amd64
-
-package-ubuntu18.04-arm64:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-arm64
-
-package-ubuntu18.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-ppc64le
-
-.buildx-setup:
-  before_script:
-    -  export BUILDX_VERSION=v0.6.3
-    -  apk add --no-cache curl
-    -  mkdir -p ~/.docker/cli-plugins
-    -  curl -sSLo ~/.docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
-    -  chmod a+x ~/.docker/cli-plugins/docker-buildx
-
-    -  docker buildx create --use --platform=linux/amd64,linux/arm64
-
-    -  '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'
-
-# Define the image build targets
-.image-build:
-  stage: image-build
-  variables:
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-    PUSH_ON_BUILD: "true"
-  before_script:
-    - !reference [.buildx-setup, before_script]
-
-    - apk add --no-cache bash make git
-    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-  script:
-    - make -f deployments/container/Makefile build-${DIST}
-
-image-ubi8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubi8
-  needs:
-    # Note: The ubi8 image uses the centos7 packages
-    - package-centos7-aarch64
-    - package-centos7-x86_64
-
-image-ubuntu20.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu20.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - job: package-ubuntu18.04-ppc64le
-      optional: true
-
-# The DIST=packaging target creates an image containing all built packages
-image-packaging:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-packaging
-  needs:
-    - job: package-ubuntu18.04-amd64
-    - job: package-ubuntu18.04-arm64
-    - job: package-amazonlinux2-aarch64
-      optional: true
-    - job: package-amazonlinux2-x86_64
-      optional: true
-    - job: package-centos7-aarch64
-      optional: true
-    - job: package-centos7-x86_64
-      optional: true
-    - job: package-centos8-ppc64le
-      optional: true
-    - job: package-debian10-amd64
-      optional: true
-    - job: package-opensuse-leap15.1-x86_64
-      optional: true
-    - job: package-ubuntu18.04-ppc64le
-      optional: true
-
-# Define publish test helpers
-.test:docker:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "docker"
-
-.test:containerd:
-  # TODO: The containerd tests fail due to issues with SIGHUP.
-  # Until this is resolved with retry up to twice and allow failure here.
-  retry: 2
-  allow_failure: true
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "containerd"
-
-.test:crio:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "crio"
-
-# Define the test targets
-test-toolkit-ubuntu20.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-containerd-ubuntu20.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-crio-ubuntu20.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-docker-ubuntu20.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
--- a/.nvidia-ci.yml
+++ b/.nvidia-ci.yml
@@ -39,19 +39,62 @@ variables:
  KITMAKER_RELEASE_FOLDER: "kitmaker"
  PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"

-.image-pull:
-  stage: image-build
+# .copy-images copies the required application and packaging images from the
+# IN_IMAGE="${IN_IMAGE_NAME}:${IN_IMAGE_TAG}${TAG_SUFFIX}"
+# to
+# OUT_IMAGE="${OUT_IMAGE_NAME}:${OUT_IMAGE_TAG}${TAG_SUFFIX}"
+# The script also logs into IN_REGISTRY and OUT_REGISTRY using the supplied
+# username and tokens.
+.copy-images:
+  parallel:
+    matrix:
+      - TAG_SUFFIX: ["", "-packaging"]
+  before_script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache make bash
+  variables:
+    REGCTL: regctl
+  script:
+    - |
+      if [ -n ${IN_REGISTRY} ] && [ -n ${IN_REGISTRY_USER} ]; then
+        echo "Logging in to ${IN_REGISTRY}"
+        ${REGCTL} registry login "${IN_REGISTRY}" -u "${IN_REGISTRY_USER}" -p "${IN_REGISTRY_TOKEN}" || exit 1
+      fi
+
+      if [ -n ${OUT_REGISTRY} ] && [ -n ${OUT_REGISTRY_USER} ] && [ "${IN_REGISTRY}" != "${OUT_REGISTRY}" ]; then
+        echo "Logging in to ${OUT_REGISTRY}"
+        ${REGCTL} registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" || exit 1
+      fi
+
+      export IN_IMAGE="${IN_IMAGE_NAME}:${IN_IMAGE_TAG}${TAG_SUFFIX}"
+      export OUT_IMAGE="${OUT_IMAGE_NAME}:${OUT_IMAGE_TAG}${TAG_SUFFIX}"
+
+      echo "Copying ${IN_IMAGE} to ${OUT_IMAGE}"
+      ${REGCTL} image copy ${IN_IMAGE} ${OUT_IMAGE}
+
+# pull-images pulls images from the public CI registry to the internal CI registry.
+pull-images:
+  extends:
+    - .copy-images
+  stage: pull
  variables:
    IN_REGISTRY: "${STAGING_REGISTRY}"
-    IN_IMAGE_NAME: container-toolkit
-    IN_VERSION: "${STAGING_VERSION}"
+    IN_IMAGE_NAME: ${STAGING_REGISTRY}/container-toolkit
+    IN_IMAGE_TAG: "${STAGING_VERSION}"
+
+    OUT_REGISTRY: "${CI_REGISTRY}"
    OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
    OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
-    OUT_REGISTRY: "${CI_REGISTRY}"
    OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    PUSH_MULTIPLE_TAGS: "false"
+    OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
  # We delay the job start to allow the public pipeline to generate the required images.
  rules:
+    # If the pipeline is triggered from a tag or the WEB UI we don't delay the
+    # start of the pipeline.
+    - if: $CI_COMMIT_TAG || $CI_PIPELINE_SOURCE == "web"
+    # If the pipeline is triggered through other means (i.e. a branch or MR)
+    # we add a 30 minute delay to ensure that the images are available in the
+    # public CI registry.
    - when: delayed
      start_in: 30 minutes
  timeout: 30 minutes
@@ -60,30 +103,6 @@ variables:
    when:
      - job_execution_timeout
      - stuck_or_timeout_failure
-  before_script:
-    - !reference [.regctl-setup, before_script]
-    - apk add --no-cache make bash
-    - >
-      regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
-  script:
-    - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
-    - make -f deployments/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
-
-image-ubi8:
-  extends:
-    - .dist-ubi8
-    - .image-pull
-
-image-ubuntu20.04:
-  extends:
-    - .dist-ubuntu20.04
-    - .image-pull
-
-# The DIST=packaging target creates an image containing all built packages
-image-packaging:
-  extends:
-    - .dist-packaging
-    - .image-pull

 # We skip the integration tests for the internal CI:
 .integration:
@@ -95,27 +114,37 @@ image-packaging:

 # The .scan step forms the base of the image scan operation performed before releasing
 # images.
-.scan:
+scan-images:
  stage: scan
+  needs:
+  - pull-images
  image: "${PULSE_IMAGE}"
+  parallel:
+    matrix:
+      - TAG_SUFFIX: [""]
+        PLATFORM: ["linux/amd64", "linux/arm64"]
+      - TAG_SUFFIX: "-packaging"
+        PLATFORM: "linux/amd64"
  variables:
-    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
-    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
+    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}"
+    IMAGE_ARCHIVE: "container-toolkit-${CI_JOB_ID}.tar"
  rules:
-    - if: $SKIP_SCANS != "yes"
-    - when: manual
-  before_script:
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    # TODO: We should specify the architecture here and scan all architectures
-    - docker pull --platform="${PLATFORM}" "${IMAGE}"
-    - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
-    - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
-    - >
-      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
-    - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+    - if: $IGNORE_SCANS == "yes"
+      allow_failure: true
+    - when: on_success
  script:
-    - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
-    - rm -f "${IMAGE_ARCHIVE}"
+    - |
+      docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+      export SCAN_IMAGE=${IMAGE}${TAG_SUFFIX}
+      echo "Scanning image ${SCAN_IMAGE} for ${PLATFORM}"
+      docker pull --platform="${PLATFORM}" "${SCAN_IMAGE}"
+      docker save "${SCAN_IMAGE}" -o "${IMAGE_ARCHIVE}"
+      AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
+      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
+      if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+
+      pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+      rm -f "${IMAGE_ARCHIVE}"
  artifacts:
    when: always
    expire_in: 1 week
@@ -126,62 +155,10 @@ image-packaging:
      - vulns.json
      - policy_evaluation.json

-# Define the scan targets
-scan-ubuntu20.04-amd64:
-  extends:
-    - .dist-ubuntu20.04
-    - .platform-amd64
-    - .scan
-  needs:
-    - image-ubuntu20.04
-
-scan-ubuntu20.04-arm64:
-  extends:
-    - .dist-ubuntu20.04
-    - .platform-arm64
-    - .scan
-  needs:
-    - image-ubuntu20.04
-    - scan-ubuntu20.04-amd64
-
-scan-ubi8-amd64:
-  extends:
-    - .dist-ubi8
-    - .platform-amd64
-    - .scan
-  needs:
-    - image-ubi8
-
-scan-ubi8-arm64:
-  extends:
-    - .dist-ubi8
-    - .platform-arm64
-    - .scan
-  needs:
-    - image-ubi8
-    - scan-ubi8-amd64
-
-scan-packaging:
-  extends:
-    - .dist-packaging
-    - .scan
-  needs:
-    - image-packaging
-
-# Define external release helpers
-.release:ngc:
-  extends:
-    - .release:external
-  variables:
-    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
-    OUT_REGISTRY: "${NGC_REGISTRY}"
-    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
-
-.release:packages:
+upload-kitmaker-packages:
  stage: release
  needs:
-    - image-packaging
+    - pull-images
  variables:
    VERSION: "${CI_COMMIT_SHORT_SHA}"
    PACKAGE_REGISTRY: "${CI_REGISTRY}"
@@ -199,34 +176,81 @@ scan-packaging:
    - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
    - rm -rf ${ARTIFACTS_DIR}

-# Define the package release targets
-release:packages:kitmaker:
+push-images-to-staging:
  extends:
-    - .release:packages
-
-release:staging-ubuntu20.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu20.04
+    - .copy-images
+  stage: release
  needs:
-    - image-ubuntu20.04
+    - scan-images
+  variables:
+    IN_REGISTRY: "${CI_REGISTRY}"
+    IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"

-# Define the external release targets
-# Release to NGC
-release:ngc-ubuntu20.04:
-  extends:
-    - .dist-ubuntu20.04
-    - .release:ngc
+    OUT_REGISTRY: "${NGC_REGISTRY}"
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
+    OUT_IMAGE_NAME: "${NGC_STAGING_REGISTRY}/container-toolkit"
+    OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"

-release:ngc-ubi8:
+.release-images:
  extends:
-    - .dist-ubi8
-    - .release:ngc
+    - .copy-images
+  stage: release
+  needs:
+    - scan-images
+    - push-images-to-staging
+  variables:
+    IN_REGISTRY: "${CI_REGISTRY}"
+    IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"

-release:ngc-packaging:
+    OUT_REGISTRY: "${NGC_REGISTRY}"
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
+    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    OUT_IMAGE_TAG: "${CI_COMMIT_TAG}"
+
+release-images-to-ngc:
  extends:
-    - .dist-packaging
-    - .release:ngc
+    - .release-images
+  rules:
+    - if: $CI_COMMIT_TAG
+
+release-images-dummy:
+  extends:
+    - .release-images
+  variables:
+    REGCTL: "echo [DUMMY] regctl"
+  rules:
+    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
+
+# .sign-images forms the base of the jobs which sign images in the NGC registry.
+.sign-images:
+  stage: sign
+  image: ubuntu:latest
+  parallel:
+    matrix:
+      - TAG_SUFFIX: ["", "-packaging"]
+  variables:
+    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    IMAGE_TAG: "${CI_COMMIT_TAG}"
+    NGC_CLI: "ngc-cli/ngc"
+  before_script:
+    - !reference [.ngccli-setup, before_script]
+  script:
+    - |
+      # We ensure that the IMAGE_NAME and IMAGE_TAG is set
+      echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1
+      echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1
+
+      export IMAGE=${IMAGE_NAME}:${IMAGE_TAG}${TAG_SUFFIX}
+      echo "Signing the image ${IMAGE}"
+      ${NGC_CLI} registry image publish --source ${IMAGE} ${IMAGE} --public --discoverable --allow-guest --sign --org nvidia

 # Define the external image signing steps for NGC
 # Download the ngc cli binary for use in the sign steps
@@ -244,45 +268,24 @@ release:ngc-packaging:
    - unzip ngccli_linux.zip
    - chmod u+x ngc-cli/ngc

-# .sign forms the base of the deployment jobs which signs images in the CI registry.
-# This is extended with the image name and version to be deployed.
-.sign:ngc:
-  image: ubuntu:latest
-  stage: sign
+sign-ngc-images:
+  extends:
+    - .sign-images
+  needs:
+    - release-images-to-ngc
  rules:
    - if: $CI_COMMIT_TAG
  variables:
    NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}"
-    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
-    IMAGE_TAG: "${CI_COMMIT_TAG}-${DIST}"
  retry:
    max: 2
-  before_script:
-    - !reference [.ngccli-setup, before_script]
-    # We ensure that the IMAGE_NAME and IMAGE_TAG is set
-    - 'echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1'
-    - 'echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1'
-  script:
-    - 'echo "Signing the image ${IMAGE_NAME}:${IMAGE_TAG}"'
-    - ngc-cli/ngc registry image publish --source ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:${IMAGE_TAG} --public --discoverable --allow-guest --sign --org nvidia

-sign:ngc-ubuntu20.04:
+sign-images-dummy:
  extends:
-    - .dist-ubuntu20.04
-    - .sign:ngc
+    - .sign-images
  needs:
-    - release:ngc-ubuntu20.04
-
-sign:ngc-ubi8:
-  extends:
-    - .dist-ubi8
-    - .sign:ngc
-  needs:
-    - release:ngc-ubi8
-
-sign:ngc-packaging:
-  extends:
-    - .dist-packaging
-    - .sign:ngc
-  needs:
-    - release:ngc-packaging
+    - release-images-dummy
+  variables:
+    NGC_CLI: "echo [DUMMY] ngc-cli/ngc"
+  rules:
+    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
--- a/cmd/nvidia-ctk-installer/main.go
+++ b/cmd/nvidia-ctk-installer/main.go
@@ -14,6 +14,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/toolkit"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )

 const (
@@ -36,10 +37,11 @@ var signalReceived = make(chan bool, 1)
 type options struct {
 	toolkitInstallDir string

-	noDaemon   bool
-	runtime    string
-	pidFile    string
-	sourceRoot string
+	noDaemon    bool
+	runtime     string
+	pidFile     string
+	sourceRoot  string
+	packageType string

 	toolkitOptions toolkit.Options
 	runtimeOptions runtime.Options
@@ -123,11 +125,17 @@ func (a app) build() *cli.App {
 			EnvVars:     []string{"TOOLKIT_INSTALL_DIR", "ROOT"},
 		},
 		&cli.StringFlag{
-			Name:        "source-root",
-			Value:       "/",
-			Usage:       "The folder where the required toolkit artifacts can be found",
+			Name:        "toolkit-source-root",
+			Usage:       "The folder where the required toolkit artifacts can be found. If this is not specified, the path /artifacts/{{ .ToolkitPackageType }} is used where ToolkitPackageType is the resolved package type",
 			Destination: &options.sourceRoot,
-			EnvVars:     []string{"SOURCE_ROOT"},
+			EnvVars:     []string{"TOOLKIT_SOURCE_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "toolkit-package-type",
+			Usage:       "specify the package type to use for the toolkit. One of ['deb', 'rpm', 'auto', '']. If 'auto' or '' are used, the type is inferred automatically.",
+			Value:       "auto",
+			Destination: &options.packageType,
+			EnvVars:     []string{"TOOLKIT_PACKAGE_TYPE"},
 		},
 		&cli.StringFlag{
 			Name:        "pid-file",
@@ -145,6 +153,15 @@ func (a app) build() *cli.App {
 }

 func (a *app) Before(c *cli.Context, o *options) error {
+	if o.sourceRoot == "" {
+		sourceRoot, err := a.resolveSourceRoot(o.runtimeOptions.HostRootMount, o.packageType)
+		if err != nil {
+			return fmt.Errorf("failed to resolve source root: %v", err)
+		}
+		a.logger.Infof("Resolved source root to %v", sourceRoot)
+		o.sourceRoot = sourceRoot
+	}
+
 	a.toolkit = toolkit.NewInstaller(
 		toolkit.WithLogger(a.logger),
 		toolkit.WithSourceRoot(o.sourceRoot),
@@ -277,3 +294,35 @@ func (a *app) shutdown(pidFile string) {
 		a.logger.Warningf("Unable to remove pidfile: %v", err)
 	}
 }
+
+func (a *app) resolveSourceRoot(hostRoot string, packageType string) (string, error) {
+	resolvedPackageType, err := a.resolvePackageType(hostRoot, packageType)
+	if err != nil {
+		return "", err
+	}
+	switch resolvedPackageType {
+	case "deb":
+		return "/artifacts/deb", nil
+	case "rpm":
+		return "/artifacts/rpm", nil
+	default:
+		return "", fmt.Errorf("invalid package type: %v", resolvedPackageType)
+	}
+}
+
+func (a *app) resolvePackageType(hostRoot string, packageType string) (rPackageTypes string, rerr error) {
+	if packageType != "" && packageType != "auto" {
+		return packageType, nil
+	}
+
+	locator := lookup.NewExecutableLocator(a.logger, hostRoot)
+	if candidates, err := locator.Locate("/usr/bin/rpm"); err == nil && len(candidates) > 0 {
+		return "rpm", nil
+	}
+
+	if candidates, err := locator.Locate("/usr/bin/dpkg"); err == nil && len(candidates) > 0 {
+		return "deb", nil
+	}
+
+	return "deb", nil
+}
--- a/cmd/nvidia-ctk-installer/main_test.go
+++ b/cmd/nvidia-ctk-installer/main_test.go
@@ -433,7 +433,7 @@ swarm-resource = ""
 				"--driver-root-ctr-path=" + hostRoot,
 				"--pid-file=" + filepath.Join(testRoot, "toolkit.pid"),
 				"--restart-mode=none",
-				"--source-root=" + filepath.Join(artifactRoot, "deb"),
+				"--toolkit-source-root=" + filepath.Join(artifactRoot, "deb"),
 			}

 			err := app.Run(append(testArgs, tc.args...))
--- a/cmd/nvidia-ctk-installer/toolkit/installer/installer.go
+++ b/cmd/nvidia-ctk-installer/toolkit/installer/installer.go
@@ -47,7 +47,9 @@ var _ Installer = (*toolkitInstaller)(nil)

 // New creates a toolkit installer with the specified options.
 func New(opts ...Option) (Installer, error) {
-	t := &toolkitInstaller{}
+	t := &toolkitInstaller{
+		sourceRoot: "/",
+	}
 	for _, opt := range opts {
 		opt(t)
 	}
@@ -55,9 +57,6 @@ func New(opts ...Option) (Installer, error) {
 	if t.logger == nil {
 		t.logger = logger.New()
 	}
-	if t.sourceRoot == "" {
-		t.sourceRoot = "/"
-	}
 	if t.artifactRoot == nil {
 		artifactRoot, err := newArtifactRoot(t.logger, t.sourceRoot)
 		if err != nil {
--- a/cmd/nvidia-ctk-installer/toolkit/toolkit.go
+++ b/cmd/nvidia-ctk-installer/toolkit/toolkit.go
@@ -215,7 +215,8 @@ func Flags(opts *Options) []cli.Flag {

 // An Installer is used to install the NVIDIA Container Toolkit from the toolkit container.
 type Installer struct {
-	logger     logger.Interface
+	logger logger.Interface
+
 	sourceRoot string
 	// toolkitRoot specifies the destination path at which the toolkit is installed.
 	toolkitRoot string
--- a/deployments/container/Dockerfile
+++ b/deployments/container/Dockerfile
@@ -0,0 +1,149 @@
+# SPDX-FileCopyrightText: Copyright (c) 2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 AS build
+
+RUN dnf install -y \
+    wget make git gcc \
+     && \
+    rm -rf /var/cache/yum/*
+
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64 | arm64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+
+ENV GOPATH=/go
+ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
+
+WORKDIR /build
+COPY . .
+
+RUN mkdir -p /artifacts/bin
+ARG VERSION="N/A"
+ARG GIT_COMMIT="unknown"
+RUN make PREFIX=/artifacts/bin cmd-nvidia-ctk-installer
+
+# The packaging stage collects the deb and rpm packages built for supported
+# architectures.
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 AS packaging
+
+ARG ARTIFACTS_ROOT
+COPY ${ARTIFACTS_ROOT} /artifacts/packages/
+
+WORKDIR /artifacts/packages
+
+# build-args are added to the manifest.txt file below.
+ARG PACKAGE_VERSION
+ARG GIT_BRANCH
+ARG GIT_COMMIT
+ARG GIT_COMMIT_SHORT
+ARG SOURCE_DATE_EPOCH
+ARG VERSION
+
+# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
+RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
+    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
+    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+
+# The debpackages stage is used to extract the contents of deb packages.
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubuntu20.04 AS debpackages
+
+ARG TARGETARCH
+ARG PACKAGE_DIST_DEB=ubuntu18.04
+
+COPY --from=packaging /artifacts/packages/${PACKAGE_DIST_DEB} /deb-packages
+
+RUN mkdir -p /artifacts/deb
+RUN set -eux; \
+    \
+    case "${TARGETARCH}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64 | arm64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    for p in $(ls /deb-packages/${ARCH}/*.deb); do dpkg-deb -xv $p /artifacts/deb/; done
+
+# The rpmpackages stage is used to extract the contents of the rpm packages.
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 AS rpmpackages
+RUN dnf install -y cpio
+
+ARG TARGETARCH
+ARG PACKAGE_DIST_RPM=centos7
+
+COPY --from=packaging /artifacts/packages/${PACKAGE_DIST_RPM} /rpm-packages
+
+RUN mkdir -p /artifacts/rpm
+RUN set -eux; \
+    \
+    case "${TARGETARCH}" in \
+        x86_64 | amd64) ARCH='x86_64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64 | arm64) ARCH='aarch64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    for p in $(ls /rpm-packages/${ARCH}/*.rpm); do rpm2cpio $p | cpio -idmv -D /artifacts/rpm; done
+
+# The artifacts image serves as an intermediate stage to collect the artifacts
+# From the previous stages:
+# - The extracted deb packages
+# - The extracted rpm packages
+# - The nvidia-ctk-installer binary
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 AS artifacts
+
+COPY --from=rpmpackages /artifacts/rpm /artifacts/rpm
+COPY --from=debpackages /artifacts/deb /artifacts/deb
+COPY --from=build /artifacts/bin /artifacts/build
+
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=void
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+COPY --from=artifacts /artifacts/rpm /artifacts/rpm
+COPY --from=artifacts /artifacts/deb /artifacts/deb
+COPY --from=artifacts /artifacts/build /work
+
+WORKDIR /work
+ENV PATH=/work:$PATH
+
+ARG VERSION
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+
+ENTRYPOINT ["/work/nvidia-ctk-installer"]
--- a/deployments/container/Dockerfile.packaging
+++ b/deployments/container/Dockerfile.packaging
@@ -1,38 +0,0 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG GOLANG_VERSION=x.x.x
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubuntu20.04
-
-ARG ARTIFACTS_ROOT
-COPY ${ARTIFACTS_ROOT} /artifacts/packages/
-
-WORKDIR /artifacts/packages
-
-# build-args are added to the manifest.txt file below.
-ARG PACKAGE_DIST
-ARG PACKAGE_VERSION
-ARG GIT_BRANCH
-ARG GIT_COMMIT
-ARG GIT_COMMIT_SHORT
-ARG SOURCE_DATE_EPOCH
-ARG VERSION
-
-# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
-RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
-    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
-    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
--- a/deployments/container/Dockerfile.ubi8
+++ b/deployments/container/Dockerfile.ubi8
@@ -1,90 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi8 AS build
-
-RUN yum install -y \
-    wget make git gcc \
-     && \
-    rm -rf /var/cache/yum/*
-
-ARG GOLANG_VERSION=x.x.x
-RUN set -eux; \
-    \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-        x86_64 | amd64) ARCH='amd64' ;; \
-        ppc64el | ppc64le) ARCH='ppc64le' ;; \
-        aarch64 | arm64) ARCH='arm64' ;; \
-        *) echo "unsupported architecture" ; exit 1 ;; \
-    esac; \
-    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
-    | tar -C /usr/local -xz
-
-
-ENV GOPATH=/go
-ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
-
-WORKDIR /build
-COPY . .
-
-RUN mkdir /artifacts
-ARG VERSION="N/A"
-ARG GIT_COMMIT="unknown"
-RUN make PREFIX=/artifacts cmd-nvidia-ctk-installer
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi8
-
-ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=void
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
-
-ARG ARTIFACTS_ROOT
-ARG PACKAGE_DIST
-COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
-
-WORKDIR /artifacts/packages
-
-ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH=${TARGETARCH}
-
-RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
-    yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-1.*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-1.*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*-${PACKAGE_VERSION}*.rpm
-
-WORKDIR /work
-
-COPY --from=build /artifacts/nvidia-ctk-installer /work/nvidia-ctk-installer
-RUN ln -s nvidia-ctk-installer nvidia-toolkit
-
-ENV PATH=/work:$PATH
-
-ARG VERSION
-LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
-LABEL name="NVIDIA Container Runtime Config"
-LABEL vendor="NVIDIA"
-LABEL version="${VERSION}"
-LABEL release="N/A"
-LABEL summary="Automatically Configure your Container Runtime for GPU support."
-LABEL description="See summary"
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
-
-ENTRYPOINT ["/work/nvidia-ctk-installer"]
--- a/deployments/container/Dockerfile.ubuntu
+++ b/deployments/container/Dockerfile.ubuntu
@@ -1,98 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubuntu20.04 AS build
-
-RUN apt-get update && \
-    apt-get install -y wget make git gcc \
-    && \
-    rm -rf /var/lib/apt/lists/*
-
-ARG GOLANG_VERSION=x.x.x
-RUN set -eux; \
-    \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-        x86_64 | amd64) ARCH='amd64' ;; \
-        ppc64el | ppc64le) ARCH='ppc64le' ;; \
-        aarch64 | arm64) ARCH='arm64' ;; \
-        *) echo "unsupported architecture" ; exit 1 ;; \
-    esac; \
-    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
-    | tar -C /usr/local -xz
-
-ENV GOPATH=/go
-ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
-
-WORKDIR /build
-COPY . .
-
-RUN mkdir /artifacts
-ARG VERSION="N/A"
-ARG GIT_COMMIT="unknown"
-RUN make PREFIX=/artifacts cmd-nvidia-ctk-installer
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubuntu20.04
-
-# Remove the CUDA repository configurations to avoid issues with rotated GPG keys
-RUN rm -f /etc/apt/sources.list.d/cuda.list
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libcap2 \
-    curl \
-        && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=void
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
-
-ARG ARTIFACTS_ROOT
-ARG PACKAGE_DIST
-COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
-
-WORKDIR /artifacts/packages
-
-ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH=${TARGETARCH}
-
-RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_1.*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_1.*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*_${PACKAGE_VERSION}*.deb
-
-WORKDIR /work
-
-COPY --from=build /artifacts/nvidia-ctk-installer /work/nvidia-ctk-installer
-RUN ln -s nvidia-ctk-installer nvidia-toolkit
-
-ENV PATH=/work:$PATH
-
-ARG VERSION
-LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
-LABEL name="NVIDIA Container Runtime Config"
-LABEL vendor="NVIDIA"
-LABEL version="${VERSION}"
-LABEL release="N/A"
-LABEL summary="Automatically Configure your Container Runtime for GPU support."
-LABEL description="See summary"
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
-
-ENTRYPOINT ["/work/nvidia-ctk-installer"]
--- a/deployments/container/Makefile
+++ b/deployments/container/Makefile
@@ -29,17 +29,17 @@ include $(CURDIR)/versions.mk

 IMAGE_VERSION := $(VERSION)

-IMAGE_TAG ?= $(VERSION)-$(DIST)
+IMAGE_TAG ?= $(VERSION)
 IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)

 OUT_IMAGE_NAME ?= $(IMAGE_NAME)
 OUT_IMAGE_VERSION ?= $(IMAGE_VERSION)
-OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
+OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)
 OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)

 ##### Public rules #####
-DEFAULT_PUSH_TARGET := ubuntu20.04
-DISTRIBUTIONS := ubuntu20.04 ubi8
+DEFAULT_PUSH_TARGET := ubi9
+DISTRIBUTIONS := $(DEFAULT_PUSH_TARGET)

 META_TARGETS := packaging

@@ -56,30 +56,16 @@ else
 include $(CURDIR)/deployments/container/multi-arch.mk
 endif

-# For the default push target we also push a short tag equal to the version.
-# We skip this for the development release
-DEVEL_RELEASE_IMAGE_VERSION ?= devel
-PUSH_MULTIPLE_TAGS ?= true
-ifeq ($(strip $(OUT_IMAGE_VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
-PUSH_MULTIPLE_TAGS = false
-endif
-ifeq ($(PUSH_MULTIPLE_TAGS),true)
-push-$(DEFAULT_PUSH_TARGET): push-short
-endif
-
-push-%: DIST = $(*)
-push-short: DIST = $(DEFAULT_PUSH_TARGET)
-
 # Define the push targets
 $(PUSH_TARGETS): push-%:
 	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)

-push-short:
-	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
+DOCKERFILE = $(CURDIR)/deployments/container/Dockerfile

-
-build-%: DIST = $(*)
-build-%: DOCKERFILE = $(CURDIR)/deployments/container/Dockerfile.$(DOCKERFILE_SUFFIX)
+# For packaging targets we set the output image tag to include the -packaging suffix.
+%-packaging: INTERMEDIATE_TARGET := --target=packaging
+%-packaging: IMAGE_TAG = $(IMAGE_VERSION)-packaging
+%-packaging: OUT_IMAGE_TAG = $(IMAGE_VERSION)-packaging

 ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))

@@ -90,10 +76,12 @@ $(IMAGE_TARGETS): image-%: $(ARTIFACTS_ROOT)
 		--provenance=false --sbom=false \
 		$(DOCKER_BUILD_OPTIONS) \
 		$(DOCKER_BUILD_PLATFORM_OPTIONS) \
+		$(INTERMEDIATE_TARGET) \
 		--tag $(IMAGE) \
 		--build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \
 		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
+		--build-arg PACKAGE_DIST_DEB="$(PACKAGE_DIST_DEB)" \
+		--build-arg PACKAGE_DIST_RPM="$(PACKAGE_DIST_RPM)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg VERSION="$(VERSION)" \
 		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
@@ -103,25 +91,19 @@ $(IMAGE_TARGETS): image-%: $(ARTIFACTS_ROOT)
 		-f $(DOCKERFILE) \
 		$(CURDIR)

-build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
-build-ubuntu%: PACKAGE_DIST = ubuntu18.04

-build-ubi8: DOCKERFILE_SUFFIX := ubi8
-build-ubi8: PACKAGE_DIST = centos7
+PACKAGE_DIST_DEB = ubuntu18.04
+# TODO: This needs to be set to centos8 for ppc64le builds
+PACKAGE_DIST_RPM = centos7

-build-packaging: DOCKERFILE_SUFFIX := packaging
-build-packaging: PACKAGE_ARCH := amd64
-build-packaging: PACKAGE_DIST = all
+# Handle the default build target.
+.PHONY: build push
+build: build-$(DEFAULT_PUSH_TARGET)
+push: push-$(DEFAULT_PUSH_TARGET)

 # Test targets
 test-%: DIST = $(*)

-# Handle the default build target.
-.PHONY: build
-build: $(DEFAULT_PUSH_TARGET)
-$(DEFAULT_PUSH_TARGET): build-$(DEFAULT_PUSH_TARGET)
-$(DEFAULT_PUSH_TARGET): DIST = $(DEFAULT_PUSH_TARGET)
-
 TEST_CASES ?= docker crio containerd
 $(TEST_TARGETS): test-%:
 	TEST_CASES="$(TEST_CASES)" bash -x $(CURDIR)/test/container/main.sh run \
--- a/deployments/container/native-only.mk
+++ b/deployments/container/native-only.mk
@@ -23,11 +23,3 @@ $(BUILD_TARGETS): build-%: image-%
 else
 $(BUILD_TARGETS): build-%: image-%
 endif
-
-# For the default distribution we also retag the image.
-# Note: This needs to be updated for multi-arch images.
-ifeq ($(IMAGE_TAG),$(VERSION)-$(DIST))
-$(DEFAULT_PUSH_TARGET):
-	$(DOCKER) image inspect $(IMAGE) > /dev/null || $(DOCKER) pull $(IMAGE)
-	$(DOCKER) tag $(IMAGE) $(subst :$(IMAGE_TAG),:$(VERSION),$(IMAGE))
-endif
--- a/go.mod
+++ b/go.mod
@@ -27,6 +27,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
+	github.com/opencontainers/cgroups v0.0.1 // indirect
 	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect
--- a/go.sum
+++ b/go.sum
@@ -37,6 +37,8 @@ github.com/moby/sys/reexec v0.1.0/go.mod h1:EqjBg8F3X7iZe5pU6nRZnYCMUTXoxsjiIfHu
 github.com/moby/sys/symlink v0.3.0 h1:GZX89mEZ9u53f97npBy4Rc3vJKj7JBDj/PN2I22GrNU=
 github.com/moby/sys/symlink v0.3.0/go.mod h1:3eNdhduHmYPcgsJtZXW1W4XUJdZGBIkttZ8xKqPUJq0=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
+github.com/opencontainers/cgroups v0.0.1 h1:MXjMkkFpKv6kpuirUa4USFBas573sSAY082B4CiHEVA=
+github.com/opencontainers/cgroups v0.0.1/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs=
 github.com/opencontainers/runc v1.3.0 h1:cvP7xbEvD0QQAs0nZKLzkVog2OPZhI/V2w3WmTmUSXI=
 github.com/opencontainers/runc v1.3.0/go.mod h1:9wbWt42gV+KRxKRVVugNP6D5+PQciRbenB4fLVsqGPs=
 github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
--- a/internal/config/image/cuda_image.go
+++ b/internal/config/image/cuda_image.go
@@ -19,6 +19,7 @@ package image
 import (
 	"fmt"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"

@@ -143,8 +144,8 @@ func (i CUDA) HasDisableRequire() bool {
 	return false
 }

-// DevicesFromEnvvars returns the devices requested by the image through environment variables
-func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
+// devicesFromEnvvars returns the devices requested by the image through environment variables
+func (i CUDA) devicesFromEnvvars(envVars ...string) []string {
 	// We concantenate all the devices from the specified env.
 	var isSet bool
 	var devices []string
@@ -165,15 +166,15 @@ func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {

 	// Environment variable unset with legacy image: default to "all".
 	if !isSet && len(devices) == 0 && i.IsLegacy() {
-		return NewVisibleDevices("all")
+		devices = []string{"all"}
 	}

 	// Environment variable unset or empty or "void": return nil
 	if len(devices) == 0 || requested["void"] {
-		return NewVisibleDevices("void")
+		devices = []string{"void"}
 	}

-	return NewVisibleDevices(devices...)
+	return NewVisibleDevices(devices...).List()
 }

 // GetDriverCapabilities returns the requested driver capabilities.
@@ -232,6 +233,22 @@ func (i CUDA) OnlyFullyQualifiedCDIDevices() bool {
 	return hasCDIdevice
 }

+// visibleEnvVars returns the environment variables that are used to determine device visibility.
+// It returns the preferred environment variables that are set, or NVIDIA_VISIBLE_DEVICES if none are set.
+func (i CUDA) visibleEnvVars() []string {
+	var envVars []string
+	for _, envVar := range i.preferredVisibleDeviceEnvVars {
+		if !i.HasEnvvar(envVar) {
+			continue
+		}
+		envVars = append(envVars, envVar)
+	}
+	if len(envVars) > 0 {
+		return envVars
+	}
+	return []string{EnvVarNvidiaVisibleDevices}
+}
+
 // VisibleDevices returns a list of devices requested in the container image.
 // If volume mount requests are enabled these are returned if requested,
 // otherwise device requests through environment variables are considered.
@@ -253,7 +270,7 @@ func (i CUDA) VisibleDevices() []string {
 	}

 	// Get the Fallback to reading from the environment variable if privileges are correct
-	envVarDeviceRequests := i.VisibleDevicesFromEnvVar()
+	envVarDeviceRequests := i.visibleDevicesFromEnvVar()
 	if len(envVarDeviceRequests) == 0 {
 		return nil
 	}
@@ -265,7 +282,10 @@ func (i CUDA) VisibleDevices() []string {
 	}

 	// We log a warning if we are ignoring the environment variable requests.
-	i.logger.Warningf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES in unprivileged container")
+	envVars := i.visibleEnvVars()
+	if len(envVars) > 0 {
+		i.logger.Warningf("Ignoring devices requested by environment variable(s) in unprivileged container: %v", envVars)
+	}

 	return nil
 }
@@ -281,31 +301,34 @@ func (i CUDA) cdiDeviceRequestsFromAnnotations() []string {
 		return nil
 	}

-	var devices []string
-	for key, value := range i.annotations {
+	var annotationKeys []string
+	for key := range i.annotations {
 		for _, prefix := range i.annotationsPrefixes {
 			if strings.HasPrefix(key, prefix) {
-				devices = append(devices, strings.Split(value, ",")...)
+				annotationKeys = append(annotationKeys, key)
 				// There is no need to check additional prefixes since we
 				// typically deduplicate devices in any case.
 				break
 			}
 		}
 	}
+	// We sort the annotationKeys for consistent results.
+	slices.Sort(annotationKeys)
+
+	var devices []string
+	for _, key := range annotationKeys {
+		devices = append(devices, strings.Split(i.annotations[key], ",")...)
+	}
 	return devices
 }

-// VisibleDevicesFromEnvVar returns the set of visible devices requested through environment variables.
+// visibleDevicesFromEnvVar returns the set of visible devices requested through environment variables.
 // If any of the preferredVisibleDeviceEnvVars are present in the image, they
 // are used to determine the visible devices. If this is not the case, the
 // NVIDIA_VISIBLE_DEVICES environment variable is used.
-func (i CUDA) VisibleDevicesFromEnvVar() []string {
-	for _, envVar := range i.preferredVisibleDeviceEnvVars {
-		if i.HasEnvvar(envVar) {
-			return i.DevicesFromEnvvars(i.preferredVisibleDeviceEnvVars...).List()
-		}
-	}
-	return i.DevicesFromEnvvars(EnvVarNvidiaVisibleDevices).List()
+func (i CUDA) visibleDevicesFromEnvVar() []string {
+	envVars := i.visibleEnvVars()
+	return i.devicesFromEnvvars(envVars...)
 }

 // visibleDevicesFromMounts returns the set of visible devices requested as mounts.
@@ -391,7 +414,7 @@ func (m cdiDeviceMountRequest) qualifiedName() (string, error) {

 // ImexChannelsFromEnvVar returns the list of IMEX channels requested for the image.
 func (i CUDA) ImexChannelsFromEnvVar() []string {
-	imexChannels := i.DevicesFromEnvvars(EnvVarNvidiaImexChannels).List()
+	imexChannels := i.devicesFromEnvvars(EnvVarNvidiaImexChannels)
 	if len(imexChannels) == 1 && imexChannels[0] == "all" {
 		return nil
 	}
--- a/internal/config/image/cuda_image_test.go
+++ b/internal/config/image/cuda_image_test.go
@@ -429,7 +429,7 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			)

 			require.NoError(t, err)
-			devices := image.VisibleDevicesFromEnvVar()
+			devices := image.visibleDevicesFromEnvVar()
 			require.EqualValues(t, tc.expectedDevices, devices)
 		})
 	}
@@ -508,13 +508,15 @@ func TestGetVisibleDevicesFromMounts(t *testing.T) {

 func TestVisibleDevices(t *testing.T) {
 	var tests = []struct {
-		description        string
-		mountDevices       []specs.Mount
-		envvarDevices      string
-		privileged         bool
-		acceptUnprivileged bool
-		acceptMounts       bool
-		expectedDevices    []string
+		description                   string
+		mountDevices                  []specs.Mount
+		envvarDevices                 string
+		privileged                    bool
+		acceptUnprivileged            bool
+		acceptMounts                  bool
+		preferredVisibleDeviceEnvVars []string
+		env                           map[string]string
+		expectedDevices               []string
 	}{
 		{
 			description: "Mount devices, unprivileged, no accept unprivileged",
@@ -597,20 +599,92 @@ func TestVisibleDevices(t *testing.T) {
 			acceptMounts:       false,
 			expectedDevices:    nil,
 		},
+		// New test cases for visibleEnvVars functionality
+		{
+			description:                   "preferred env var set and present in env, privileged",
+			mountDevices:                  nil,
+			envvarDevices:                 "",
+			privileged:                    true,
+			acceptUnprivileged:            false,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS"},
+			env: map[string]string{
+				"DOCKER_RESOURCE_GPUS": "GPU-12345",
+			},
+			expectedDevices: []string{"GPU-12345"},
+		},
+		{
+			description:                   "preferred env var set and present in env, unprivileged but accepted",
+			mountDevices:                  nil,
+			envvarDevices:                 "",
+			privileged:                    false,
+			acceptUnprivileged:            true,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS"},
+			env: map[string]string{
+				"DOCKER_RESOURCE_GPUS": "GPU-12345",
+			},
+			expectedDevices: []string{"GPU-12345"},
+		},
+		{
+			description:                   "preferred env var set and present in env, unprivileged and not accepted",
+			mountDevices:                  nil,
+			envvarDevices:                 "",
+			privileged:                    false,
+			acceptUnprivileged:            false,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS"},
+			env: map[string]string{
+				"DOCKER_RESOURCE_GPUS": "GPU-12345",
+			},
+			expectedDevices: nil,
+		},
+		{
+			description:                   "multiple preferred env vars, both present, privileged",
+			mountDevices:                  nil,
+			envvarDevices:                 "",
+			privileged:                    true,
+			acceptUnprivileged:            false,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				"DOCKER_RESOURCE_GPUS":            "GPU-12345",
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": "GPU-67890",
+			},
+			expectedDevices: []string{"GPU-12345", "GPU-67890"},
+		},
+		{
+			description:                   "preferred env var not present, fallback to NVIDIA_VISIBLE_DEVICES, privileged",
+			mountDevices:                  nil,
+			envvarDevices:                 "GPU-12345",
+			privileged:                    true,
+			acceptUnprivileged:            false,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS"},
+			env: map[string]string{
+				EnvVarNvidiaVisibleDevices: "GPU-12345",
+			},
+			expectedDevices: []string{"GPU-12345"},
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			// Wrap the call to getDevices() in a closure.
+			// Create env map with both NVIDIA_VISIBLE_DEVICES and any additional env vars
+			env := make(map[string]string)
+			if tc.envvarDevices != "" {
+				env[EnvVarNvidiaVisibleDevices] = tc.envvarDevices
+			}
+			for k, v := range tc.env {
+				env[k] = v
+			}
+
 			image, err := New(
-				WithEnvMap(
-					map[string]string{
-						EnvVarNvidiaVisibleDevices: tc.envvarDevices,
-					},
-				),
+				WithEnvMap(env),
 				WithMounts(tc.mountDevices),
 				WithPrivileged(tc.privileged),
 				WithAcceptDeviceListAsVolumeMounts(tc.acceptMounts),
 				WithAcceptEnvvarUnprivileged(tc.acceptUnprivileged),
+				WithPreferredVisibleDevicesEnvVars(tc.preferredVisibleDeviceEnvVars...),
 			)
 			require.NoError(t, err)
 			require.Equal(t, tc.expectedDevices, image.VisibleDevices())
--- a/internal/edits/device.go
+++ b/internal/edits/device.go
@@ -20,6 +20,8 @@ import (
 	"tags.cncf.io/container-device-interface/pkg/cdi"
 	"tags.cncf.io/container-device-interface/specs-go"

+	"github.com/opencontainers/runc/libcontainer/devices"
+
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 )

@@ -43,19 +45,37 @@ func (d device) toEdits() (*cdi.ContainerEdits, error) {
 // toSpec converts a discovered Device to a CDI Spec Device. Note
 // that missing info is filled in when edits are applied by querying the Device node.
 func (d device) toSpec() (*specs.DeviceNode, error) {
+	s := d.fromPathOrDefault()
 	// The HostPath field was added in the v0.5.0 CDI specification.
 	// The cdi package uses strict unmarshalling when loading specs from file causing failures for
 	// unexpected fields.
 	// Since the behaviour for HostPath == "" and HostPath == Path are equivalent, we clear HostPath
 	// if it is equal to Path to ensure compatibility with the widest range of specs.
-	hostPath := d.HostPath
-	if hostPath == d.Path {
-		hostPath = ""
-	}
-	s := specs.DeviceNode{
-		HostPath: hostPath,
-		Path:     d.Path,
+	if s.HostPath == d.Path {
+		s.HostPath = ""
 	}

-	return &s, nil
+	return s, nil
+}
+
+// fromPathOrDefault attempts to return the returns the information about the
+// CDI device from the specified host path.
+// If this fails a minimal device is returned so that this information can be
+// queried by the container runtime such as containerd.
+func (d device) fromPathOrDefault() *specs.DeviceNode {
+	dn, err := devices.DeviceFromPath(d.HostPath, "rwm")
+	if err != nil {
+		return &specs.DeviceNode{
+			HostPath: d.HostPath,
+			Path:     d.Path,
+		}
+	}
+
+	return &specs.DeviceNode{
+		HostPath: d.HostPath,
+		Path:     d.Path,
+		Major:    dn.Major,
+		Minor:    dn.Minor,
+		FileMode: &dn.FileMode,
+	}
 }
--- a/internal/modifier/cdi_test.go
+++ b/internal/modifier/cdi_test.go
@@ -98,7 +98,7 @@ func TestDeviceRequests(t *testing.T) {
 					"another-prefix/bar": "example.com/device=baz",
 				},
 			},
-			expectedDevices: []string{"example.com/device=bar", "example.com/device=baz"},
+			expectedDevices: []string{"example.com/device=baz", "example.com/device=bar"},
 		},
 		{
 			description: "multiple matching annotations with duplicate devices",
--- a/internal/modifier/csv.go
+++ b/internal/modifier/csv.go
@@ -33,7 +33,7 @@ import (
 // NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
 // The modifications are defined by CSV MountSpecs.
 func NewCSVModifier(logger logger.Interface, cfg *config.Config, container image.CUDA) (oci.SpecModifier, error) {
-	if devices := container.VisibleDevicesFromEnvVar(); len(devices) == 0 {
+	if devices := container.VisibleDevices(); len(devices) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}
--- a/internal/modifier/gated.go
+++ b/internal/modifier/gated.go
@@ -37,7 +37,7 @@ import (
 //
 // If not devices are selected, no changes are made.
 func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image image.CUDA, driver *root.Driver, hookCreator discover.HookCreator) (oci.SpecModifier, error) {
-	if devices := image.VisibleDevicesFromEnvVar(); len(devices) == 0 {
+	if devices := image.VisibleDevices(); len(devices) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}
--- a/internal/modifier/graphics.go
+++ b/internal/modifier/graphics.go
@@ -29,9 +29,10 @@ import (

 // NewGraphicsModifier constructs a modifier that injects graphics-related modifications into an OCI runtime specification.
 // The value of the NVIDIA_DRIVER_CAPABILITIES environment variable is checked to determine if this modification should be made.
-func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, containerImage image.CUDA, driver *root.Driver, hookCreator discover.HookCreator) (oci.SpecModifier, error) {
-	if required, reason := requiresGraphicsModifier(containerImage); !required {
-		logger.Infof("No graphics modifier required: %v", reason)
+func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, container image.CUDA, driver *root.Driver, hookCreator discover.HookCreator) (oci.SpecModifier, error) {
+	devices, reason := requiresGraphicsModifier(container)
+	if len(devices) == 0 {
+		logger.Infof("No graphics modifier required; %v", reason)
 		return nil, nil
 	}

@@ -48,7 +49,7 @@ func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, containerI
 	devRoot := driver.Root
 	drmNodes, err := discover.NewDRMNodesDiscoverer(
 		logger,
-		containerImage.DevicesFromEnvvars(image.EnvVarNvidiaVisibleDevices),
+		image.NewVisibleDevices(devices...),
 		devRoot,
 		hookCreator,
 	)
@@ -64,14 +65,15 @@ func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, containerI
 }

 // requiresGraphicsModifier determines whether a graphics modifier is required.
-func requiresGraphicsModifier(cudaImage image.CUDA) (bool, string) {
-	if devices := cudaImage.VisibleDevicesFromEnvVar(); len(devices) == 0 {
-		return false, "no devices requested"
+func requiresGraphicsModifier(cudaImage image.CUDA) ([]string, string) {
+	devices := cudaImage.VisibleDevices()
+	if len(devices) == 0 {
+		return nil, "no devices requested"
 	}

 	if !cudaImage.GetDriverCapabilities().Any(image.DriverCapabilityGraphics, image.DriverCapabilityDisplay) {
-		return false, "no required capabilities requested"
+		return nil, "no required capabilities requested"
 	}

-	return true, ""
+	return devices, ""
 }
--- a/internal/modifier/graphics_test.go
+++ b/internal/modifier/graphics_test.go
@@ -26,9 +26,9 @@ import (

 func TestGraphicsModifier(t *testing.T) {
 	testCases := []struct {
-		description      string
-		envmap           map[string]string
-		expectedRequired bool
+		description     string
+		envmap          map[string]string
+		expectedDevices []string
 	}{
 		{
 			description: "empty image does not create modifier",
@@ -52,7 +52,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "all",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 		{
 			description: "devices with graphics capability creates modifier",
@@ -60,7 +60,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "graphics",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 		{
 			description: "devices with compute,graphics capability creates modifier",
@@ -68,7 +68,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "compute,graphics",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 		{
 			description: "devices with display capability creates modifier",
@@ -76,7 +76,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "display",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 		{
 			description: "devices with display,graphics capability creates modifier",
@@ -84,7 +84,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "display,graphics",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 	}

@@ -94,7 +94,7 @@ func TestGraphicsModifier(t *testing.T) {
 				image.WithEnvMap(tc.envmap),
 			)
 			required, _ := requiresGraphicsModifier(image)
-			require.EqualValues(t, tc.expectedRequired, required)
+			require.EqualValues(t, tc.expectedDevices, required)
 		})
 	}
 }
--- a/tests/e2e/Makefile
+++ b/tests/e2e/Makefile
@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
--- a/vendor/github.com/opencontainers/cgroups/LICENSE
+++ b/vendor/github.com/opencontainers/cgroups/LICENSE
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
--- a/vendor/github.com/opencontainers/cgroups/devices/config/device.go
+++ b/vendor/github.com/opencontainers/cgroups/devices/config/device.go
@@ -0,0 +1,174 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+)
+
+const (
+	Wildcard = -1
+)
+
+type Device struct {
+	Rule
+
+	// Path to the device.
+	Path string `json:"path"`
+
+	// FileMode permission bits for the device.
+	FileMode os.FileMode `json:"file_mode"`
+
+	// Uid of the device.
+	Uid uint32 `json:"uid"`
+
+	// Gid of the device.
+	Gid uint32 `json:"gid"`
+}
+
+// Permissions is a cgroupv1-style string to represent device access. It
+// has to be a string for backward compatibility reasons, hence why it has
+// methods to do set operations.
+type Permissions string
+
+const (
+	deviceRead uint = (1 << iota)
+	deviceWrite
+	deviceMknod
+)
+
+func (p Permissions) toSet() uint {
+	var set uint
+	for _, perm := range p {
+		switch perm {
+		case 'r':
+			set |= deviceRead
+		case 'w':
+			set |= deviceWrite
+		case 'm':
+			set |= deviceMknod
+		}
+	}
+	return set
+}
+
+func fromSet(set uint) Permissions {
+	var perm string
+	if set&deviceRead == deviceRead {
+		perm += "r"
+	}
+	if set&deviceWrite == deviceWrite {
+		perm += "w"
+	}
+	if set&deviceMknod == deviceMknod {
+		perm += "m"
+	}
+	return Permissions(perm)
+}
+
+// Union returns the union of the two sets of Permissions.
+func (p Permissions) Union(o Permissions) Permissions {
+	lhs := p.toSet()
+	rhs := o.toSet()
+	return fromSet(lhs | rhs)
+}
+
+// Difference returns the set difference of the two sets of Permissions.
+// In set notation, A.Difference(B) gives you A\B.
+func (p Permissions) Difference(o Permissions) Permissions {
+	lhs := p.toSet()
+	rhs := o.toSet()
+	return fromSet(lhs &^ rhs)
+}
+
+// Intersection computes the intersection of the two sets of Permissions.
+func (p Permissions) Intersection(o Permissions) Permissions {
+	lhs := p.toSet()
+	rhs := o.toSet()
+	return fromSet(lhs & rhs)
+}
+
+// IsEmpty returns whether the set of permissions in a Permissions is
+// empty.
+func (p Permissions) IsEmpty() bool {
+	return p == Permissions("")
+}
+
+// IsValid returns whether the set of permissions is a subset of valid
+// permissions (namely, {r,w,m}).
+func (p Permissions) IsValid() bool {
+	return p == fromSet(p.toSet())
+}
+
+type Type rune
+
+const (
+	WildcardDevice Type = 'a'
+	BlockDevice    Type = 'b'
+	CharDevice     Type = 'c' // or 'u'
+	FifoDevice     Type = 'p'
+)
+
+func (t Type) IsValid() bool {
+	switch t {
+	case WildcardDevice, BlockDevice, CharDevice, FifoDevice:
+		return true
+	default:
+		return false
+	}
+}
+
+func (t Type) CanMknod() bool {
+	switch t {
+	case BlockDevice, CharDevice, FifoDevice:
+		return true
+	default:
+		return false
+	}
+}
+
+func (t Type) CanCgroup() bool {
+	switch t {
+	case WildcardDevice, BlockDevice, CharDevice:
+		return true
+	default:
+		return false
+	}
+}
+
+type Rule struct {
+	// Type of device ('c' for char, 'b' for block). If set to 'a', this rule
+	// acts as a wildcard and all fields other than Allow are ignored.
+	Type Type `json:"type"`
+
+	// Major is the device's major number.
+	Major int64 `json:"major"`
+
+	// Minor is the device's minor number.
+	Minor int64 `json:"minor"`
+
+	// Permissions is the set of permissions that this rule applies to (in the
+	// cgroupv1 format -- any combination of "rwm").
+	Permissions Permissions `json:"permissions"`
+
+	// Allow specifies whether this rule is allowed.
+	Allow bool `json:"allow"`
+}
+
+func (d *Rule) CgroupString() string {
+	var (
+		major = strconv.FormatInt(d.Major, 10)
+		minor = strconv.FormatInt(d.Minor, 10)
+	)
+	if d.Major == Wildcard {
+		major = "*"
+	}
+	if d.Minor == Wildcard {
+		minor = "*"
+	}
+	return fmt.Sprintf("%c %s:%s %s", d.Type, major, minor, d.Permissions)
+}
+
+func (d *Rule) Mkdev() (uint64, error) {
+	return mkDev(d)
+}
--- a/vendor/github.com/opencontainers/cgroups/devices/config/mknod_unix.go
+++ b/vendor/github.com/opencontainers/cgroups/devices/config/mknod_unix.go
@@ -0,0 +1,14 @@
+package config
+
+import (
+	"errors"
+
+	"golang.org/x/sys/unix"
+)
+
+func mkDev(d *Rule) (uint64, error) {
+	if d.Major == Wildcard || d.Minor == Wildcard {
+		return 0, errors.New("cannot mkdev() device with wildcards")
+	}
+	return unix.Mkdev(uint32(d.Major), uint32(d.Minor)), nil
+}
--- a/vendor/github.com/opencontainers/runc/libcontainer/devices/device_deprecated.go
+++ b/vendor/github.com/opencontainers/runc/libcontainer/devices/device_deprecated.go
@@ -0,0 +1,20 @@
+package devices
+
+import "github.com/opencontainers/cgroups/devices/config"
+
+// Deprecated: use [github.com/opencontainers/cgroups/devices/config].
+const (
+	Wildcard       = config.Wildcard
+	WildcardDevice = config.WildcardDevice
+	BlockDevice    = config.BlockDevice
+	CharDevice     = config.CharDevice
+	FifoDevice     = config.FifoDevice
+)
+
+// Deprecated: use [github.com/opencontainers/cgroups/devices/config].
+type (
+	Device      = config.Device
+	Permissions = config.Permissions
+	Type        = config.Type
+	Rule        = config.Rule
+)
--- a/vendor/github.com/opencontainers/runc/libcontainer/devices/device_unix.go
+++ b/vendor/github.com/opencontainers/runc/libcontainer/devices/device_unix.go
@@ -0,0 +1,112 @@
+//go:build !windows
+
+package devices
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sys/unix"
+)
+
+// ErrNotADevice denotes that a file is not a valid linux device.
+var ErrNotADevice = errors.New("not a device node")
+
+// Testing dependencies
+var (
+	unixLstat = unix.Lstat
+	osReadDir = os.ReadDir
+)
+
+// DeviceFromPath takes the path to a device and its cgroup_permissions (which
+// cannot be easily queried) to look up the information about a linux device
+// and returns that information as a Device struct.
+func DeviceFromPath(path, permissions string) (*Device, error) {
+	var stat unix.Stat_t
+	err := unixLstat(path, &stat)
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		devType   Type
+		mode      = stat.Mode
+		devNumber = uint64(stat.Rdev) //nolint:unconvert // Rdev is uint32 on e.g. MIPS.
+		major     = unix.Major(devNumber)
+		minor     = unix.Minor(devNumber)
+	)
+	switch mode & unix.S_IFMT {
+	case unix.S_IFBLK:
+		devType = BlockDevice
+	case unix.S_IFCHR:
+		devType = CharDevice
+	case unix.S_IFIFO:
+		devType = FifoDevice
+	default:
+		return nil, ErrNotADevice
+	}
+	return &Device{
+		Rule: Rule{
+			Type:        devType,
+			Major:       int64(major),
+			Minor:       int64(minor),
+			Permissions: Permissions(permissions),
+		},
+		Path:     path,
+		FileMode: os.FileMode(mode &^ unix.S_IFMT),
+		Uid:      stat.Uid,
+		Gid:      stat.Gid,
+	}, nil
+}
+
+// HostDevices returns all devices that can be found under /dev directory.
+func HostDevices() ([]*Device, error) {
+	return GetDevices("/dev")
+}
+
+// GetDevices recursively traverses a directory specified by path
+// and returns all devices found there.
+func GetDevices(path string) ([]*Device, error) {
+	files, err := osReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []*Device
+	for _, f := range files {
+		switch {
+		case f.IsDir():
+			switch f.Name() {
+			// ".lxc" & ".lxd-mounts" added to address https://github.com/lxc/lxd/issues/2825
+			// ".udev" added to address https://github.com/opencontainers/runc/issues/2093
+			case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts", ".udev":
+				continue
+			default:
+				sub, err := GetDevices(filepath.Join(path, f.Name()))
+				if err != nil {
+					return nil, err
+				}
+
+				out = append(out, sub...)
+				continue
+			}
+		case f.Name() == "console":
+			continue
+		}
+		device, err := DeviceFromPath(filepath.Join(path, f.Name()), "rwm")
+		if err != nil {
+			if errors.Is(err, ErrNotADevice) {
+				continue
+			}
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, err
+		}
+		if device.Type == FifoDevice {
+			continue
+		}
+		out = append(out, device)
+	}
+	return out, nil
+}
--- a/vendor/modules.txt
+++ b/vendor/modules.txt
@@ -37,8 +37,12 @@ github.com/moby/sys/reexec
 # github.com/moby/sys/symlink v0.3.0
 ## explicit; go 1.17
 github.com/moby/sys/symlink
+# github.com/opencontainers/cgroups v0.0.1
+## explicit; go 1.23.0
+github.com/opencontainers/cgroups/devices/config
 # github.com/opencontainers/runc v1.3.0
 ## explicit; go 1.23.0
+github.com/opencontainers/runc/libcontainer/devices
 github.com/opencontainers/runc/libcontainer/exeseal
 github.com/opencontainers/runc/libcontainer/system
 github.com/opencontainers/runc/libcontainer/utils
Author	SHA1	Message	Date
Evan Lezar	85adc3fd08	Extract device information from host path if possible Some checks failed CI Pipeline / code-scanning (push) Has been cancelled Details CI Pipeline / variables (push) Has been cancelled Details CI Pipeline / golang (push) Has been cancelled Details CI Pipeline / image (push) Has been cancelled Details CI Pipeline / e2e-test (push) Has been cancelled Details This change extracts the required information from the host device node if available. Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-19 15:20:31 +02:00
Evan Lezar	614e469dac	Merge pull request #1153 from elezar/switch-to-ubi9 Switch to cuda ubi9 base image	2025-06-18 13:10:32 +02:00
Evan Lezar	aafe4d7ad0	Switch to cuda ubi9 base image Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-18 11:58:23 +02:00
Evan Lezar	1f7c7ffec2	Merge pull request #1152 from elezar/remove-dist-tag Update image tags for single image	2025-06-18 11:56:14 +02:00
Evan Lezar	7e1beb7aa6	[no-relnote] Update gitlab CI for new image names We now release all images with vX.Y.Z and vX.Y.Z-packaging tags. This change updates the gitlab CI to allow for this. Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-18 11:51:52 +02:00
Evan Lezar	d560888f1f	Use single version tag for image Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-17 19:46:42 +02:00
Evan Lezar	81fb7bb9c1	Merge pull request #602 from elezar/use-ubi8-image Use single image for deb and rpm-based systems	2025-06-17 15:09:47 +02:00
Evan Lezar	208896d87d	Merge pull request #1130 from ArangoGutierrez/fix/1049 BUGFIX: modifier: respect GPU volume-mount device requests	2025-06-17 15:04:22 +02:00
Evan Lezar	82b62898bf	[no-relnote] Make devicesFromEnvvars private Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-17 11:55:00 +02:00
Carlos Eduardo Arango Gutierrez	d03a06029a	BUGFIX: modifier: respect GPU volume-mount device requests The gated modifiers used to add support for GDS, Mofed, and CUDA Forward Comatibility only check the NVIDIA_VISIBLE_DEVICES envvar to determine whether GPUs are requested and modifications should be made. This means that use cases where volume mounts are used to request devices are not supported. This change ensures that device extraction is consistent for all use cases. Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com> Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-17 11:55:00 +02:00
Evan Lezar	f4f7da65f1	Ensure consistent sorting of annotation devices Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-17 11:55:00 +02:00
Carlos Eduardo Arango Gutierrez	5fe7b06514	[no-relnote] new helper func visibleEnvVars Signed-off-by: Carlos Eduardo Arango Gutierrez <eduardoa@nvidia.com> Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-17 11:54:53 +02:00
Evan Lezar	5606caa5af	Extract deb and rpm packages to single image This change swithces to using a single image for the NVIDIA Container Toolkit contianer. Here the contents of the architecture-specific deb and rpm packages are extracted to a known root. These contents can then be installed using the updated installation mechanism which has been updated to detect the source root based on the packaging type. Signed-off-by: Evan Lezar <elezar@nvidia.com>	2025-06-16 19:15:56 +02:00