Extract device information from host path if possible

This change extracts the required information from the host device
node if available.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

.common-ci.yml

@@ -22,15 +22,7 @@ variables:
   BUILD_MULTI_ARCH_IMAGES: "true"
 
 stages:
-  - trigger
+  - pull
-  - image
-  - lint
-  - go-checks
-  - go-build
-  - unit-tests
-  - package-build
-  - image-build
-  - test
   - scan
   - release
   - sign
@@ -53,108 +45,6 @@ workflow:
     # We then add all the regular triggers
     - !reference [.pipeline-trigger-rules, rules]
 
-# The main or manual job is used to filter out distributions or architectures that are not required on
-# every build.
-.main-or-manual:
-  rules:
-    - !reference [.pipeline-trigger-rules, rules]
-    - if: $CI_PIPELINE_SOURCE == "schedule"
-      when: manual
-
-# The trigger-pipeline job adds a manualy triggered job to the pipeline on merge requests.
-trigger-pipeline:
-  stage: trigger
-  script:
-    - echo "starting pipeline"
-  rules:
-    - !reference [.main-or-manual, rules]
-    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-      when: manual
-      allow_failure: false
-    - when: always
-
-# Define the distribution targets
-.dist-centos7:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: centos7
-
-.dist-centos8:
-  variables:
-    DIST: centos8
-
-.dist-ubi8:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    DIST: ubi8
-
-.dist-ubuntu18.04:
-  variables:
-    DIST: ubuntu18.04
-
-.dist-ubuntu20.04:
-  variables:
-    DIST: ubuntu20.04
-
-.dist-packaging:
-  variables:
-    DIST: packaging
-
-# Define architecture targets
-.arch-aarch64:
-  variables:
-    ARCH: aarch64
-
-.arch-amd64:
-  variables:
-    ARCH: amd64
-
-.arch-arm64:
-  variables:
-    ARCH: arm64
-
-.arch-ppc64le:
-  rules:
-    - !reference [.main-or-manual, rules]
-  variables:
-    ARCH: ppc64le
-
-.arch-x86_64:
-  variables:
-    ARCH: x86_64
-
-# Define the platform targets
-.platform-amd64:
-  variables:
-    PLATFORM: linux/amd64
-
-.platform-arm64:
-  variables:
-    PLATFORM: linux/arm64
-
-# Define test helpers
-.integration:
-  stage: test
-  variables:
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - apk add --no-cache make bash jq
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    - docker pull "${IMAGE_NAME}:${VERSION}-${DIST}"
-  script:
-    - make -f deployments/container/Makefile test-${DIST}
-
-# Define the test targets
-test-packaging:
-  extends:
-    - .integration
-    - .dist-packaging
-  needs:
-    - image-packaging
-
 # Download the regctl binary for use in the release steps
 .regctl-setup:
   before_script:
@@ -164,84 +54,3 @@ test-packaging:
     - curl -sSLo bin/regctl https://github.com/regclient/regclient/releases/download/${REGCTL_VERSION}/regctl-linux-amd64
     - chmod a+x bin/regctl
     - export PATH=$(pwd)/bin:${PATH}
-
-# .release forms the base of the deployment jobs which push images to the CI registry.
-# This is extended with the version to be deployed (e.g. the SHA or TAG) and the
-# target os.
-.release:
-  stage: release
-  variables:
-    # Define the source image for the release
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-    # OUT_IMAGE_VERSION is overridden for external releases
-    OUT_IMAGE_VERSION: "${CI_COMMIT_SHORT_SHA}"
-  before_script:
-    - !reference [.regctl-setup, before_script]
-    # We ensure that the components of the output image are set:
-    - 'echo Image Name: ${OUT_IMAGE_NAME} ; [[ -n "${OUT_IMAGE_NAME}" ]] || exit 1'
-    - 'echo Version: ${OUT_IMAGE_VERSION} ; [[ -n "${OUT_IMAGE_VERSION}" ]] || exit 1'
-
-    - apk add --no-cache make bash
-  script:
-    # Log in to the "output" registry, tag the image and push the image
-    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
-    - regctl registry login "${CI_REGISTRY}" -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}"
-    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || echo "Logging in to output registry ${OUT_REGISTRY}"'
-    - '[ ${CI_REGISTRY} = ${OUT_REGISTRY} ] || regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"'
-
-    # Since OUT_IMAGE_NAME and OUT_IMAGE_VERSION are set, this will push the CI image to the
-    # Target
-    - make -f deployments/container/Makefile push-${DIST}
-
-# Define a staging release step that pushes an image to an internal "staging" repository
-# This is triggered for all pipelines (i.e. not only tags) to test the pipeline steps
-# outside of the release process.
-.release:staging:
-  extends:
-    - .release
-  variables:
-    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
-    OUT_REGISTRY: "${NGC_REGISTRY}"
-    OUT_IMAGE_NAME: "${NGC_REGISTRY_STAGING_IMAGE_NAME}"
-
-# Define an external release step that pushes an image to an external repository.
-# This includes a devlopment image off main.
-.release:external:
-  extends:
-    - .release
-  variables:
-    FORCE_PUBLISH_IMAGES: "yes"
-  rules:
-    - if: $CI_COMMIT_TAG
-      variables:
-        OUT_IMAGE_VERSION: "${CI_COMMIT_TAG}"
-    - if: $CI_COMMIT_BRANCH == $RELEASE_DEVEL_BRANCH
-      variables:
-        OUT_IMAGE_VERSION: "${DEVEL_RELEASE_IMAGE_VERSION}"
-
-# Define the release jobs
-release:staging-ubi8:
-  extends:
-    - .release:staging
-    - .dist-ubi8
-  needs:
-    - image-ubi8
-
-release:staging-ubuntu20.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu20.04
-  needs:
-    - test-toolkit-ubuntu20.04
-    - test-containerd-ubuntu20.04
-    - test-crio-ubuntu20.04
-    - test-docker-ubuntu20.04
-
-release:staging-packaging:
-  extends:
-    - .release:staging
-    - .dist-packaging
-  needs:
-    - test-packaging

.github/workflows/e2e.yaml

@@ -72,7 +72,7 @@ jobs:
         env:
           E2E_INSTALL_CTK: "true"
           E2E_IMAGE_NAME: ghcr.io/nvidia/container-toolkit
-          E2E_IMAGE_TAG: ${{ inputs.version }}-ubuntu20.04
+          E2E_IMAGE_TAG: ${{ inputs.version }}
           E2E_SSH_USER: ${{ secrets.E2E_SSH_USER }}
           E2E_SSH_HOST: ${{ steps.holodeck_public_dns_name.outputs.result }}
         run: |

.github/workflows/image.yaml

@@ -80,14 +80,8 @@ jobs:
     strategy:
       matrix:
         dist:
-          - ubuntu20.04
+          - ubi9
-          - ubi8
           - packaging
-        ispr:
-          - ${{ github.ref_name != 'main' && !startsWith( github.ref_name, 'release-' ) }}
-        exclude:
-          - ispr: true
-            dist: ubi8
     needs: packages
     steps:
       - uses: actions/checkout@v4

.gitlab-ci.yml

@@ -1,228 +0,0 @@
-# Copyright (c) 2019-2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-include:
-  - .common-ci.yml
-
-# Define the package build helpers
-.multi-arch-build:
-  before_script:
-    - apk add --no-cache coreutils build-base sed git bash make
-    - '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes -c yes'
-
-.package-artifacts:
-  variables:
-    ARTIFACTS_NAME: "toolkit-container-${CI_PIPELINE_ID}"
-    ARTIFACTS_ROOT: "toolkit-container-${CI_PIPELINE_ID}"
-    DIST_DIR: ${CI_PROJECT_DIR}/${ARTIFACTS_ROOT}
-
-.package-build:
-  extends:
-    - .multi-arch-build
-    - .package-artifacts
-  stage: package-build
-  timeout: 3h
-  script:
-    - ./scripts/build-packages.sh ${DIST}-${ARCH}
-
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_ROOT}
-  needs:
-    - job: package-meta-packages
-      artifacts: true
-
-# Define the package build targets
-package-meta-packages:
-  extends:
-    - .package-artifacts
-  stage: package-build
-  variables:
-    SKIP_LIBNVIDIA_CONTAINER: "yes"
-    SKIP_NVIDIA_CONTAINER_TOOLKIT: "yes"
-  parallel:
-    matrix:
-      - PACKAGING: [deb, rpm]
-  before_script:
-    - apk add --no-cache coreutils build-base sed git bash make
-  script:
-    - ./scripts/build-packages.sh ${PACKAGING}
-  artifacts:
-    name: ${ARTIFACTS_NAME}
-    paths:
-      - ${ARTIFACTS_ROOT}
-
-package-centos7-aarch64:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-aarch64
-
-package-centos7-x86_64:
-  extends:
-    - .package-build
-    - .dist-centos7
-    - .arch-x86_64
-
-package-centos8-ppc64le:
-  extends:
-    - .package-build
-    - .dist-centos8
-    - .arch-ppc64le
-
-package-ubuntu18.04-amd64:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-amd64
-
-package-ubuntu18.04-arm64:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-arm64
-
-package-ubuntu18.04-ppc64le:
-  extends:
-    - .package-build
-    - .dist-ubuntu18.04
-    - .arch-ppc64le
-
-.buildx-setup:
-  before_script:
-    -  export BUILDX_VERSION=v0.6.3
-    -  apk add --no-cache curl
-    -  mkdir -p ~/.docker/cli-plugins
-    -  curl -sSLo ~/.docker/cli-plugins/docker-buildx "https://github.com/docker/buildx/releases/download/${BUILDX_VERSION}/buildx-${BUILDX_VERSION}.linux-amd64"
-    -  chmod a+x ~/.docker/cli-plugins/docker-buildx
-
-    -  docker buildx create --use --platform=linux/amd64,linux/arm64
-
-    -  '[[ -n "${SKIP_QEMU_SETUP}" ]] || docker run --rm --privileged multiarch/qemu-user-static --reset -p yes'
-
-# Define the image build targets
-.image-build:
-  stage: image-build
-  variables:
-    IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    VERSION: "${CI_COMMIT_SHORT_SHA}"
-    PUSH_ON_BUILD: "true"
-  before_script:
-    - !reference [.buildx-setup, before_script]
-
-    - apk add --no-cache bash make git
-    - 'echo "Logging in to CI registry ${CI_REGISTRY}"'
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-  script:
-    - make -f deployments/container/Makefile build-${DIST}
-
-image-ubi8:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubi8
-  needs:
-    # Note: The ubi8 image uses the centos7 packages
-    - package-centos7-aarch64
-    - package-centos7-x86_64
-
-image-ubuntu20.04:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-ubuntu20.04
-  needs:
-    - package-ubuntu18.04-amd64
-    - package-ubuntu18.04-arm64
-    - job: package-ubuntu18.04-ppc64le
-      optional: true
-
-# The DIST=packaging target creates an image containing all built packages
-image-packaging:
-  extends:
-    - .image-build
-    - .package-artifacts
-    - .dist-packaging
-  needs:
-    - job: package-ubuntu18.04-amd64
-    - job: package-ubuntu18.04-arm64
-    - job: package-amazonlinux2-aarch64
-      optional: true
-    - job: package-amazonlinux2-x86_64
-      optional: true
-    - job: package-centos7-aarch64
-      optional: true
-    - job: package-centos7-x86_64
-      optional: true
-    - job: package-centos8-ppc64le
-      optional: true
-    - job: package-debian10-amd64
-      optional: true
-    - job: package-opensuse-leap15.1-x86_64
-      optional: true
-    - job: package-ubuntu18.04-ppc64le
-      optional: true
-
-# Define publish test helpers
-.test:docker:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "docker"
-
-.test:containerd:
-  # TODO: The containerd tests fail due to issues with SIGHUP.
-  # Until this is resolved with retry up to twice and allow failure here.
-  retry: 2
-  allow_failure: true
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "containerd"
-
-.test:crio:
-  extends:
-    - .integration
-  variables:
-    TEST_CASES: "crio"
-
-# Define the test targets
-test-toolkit-ubuntu20.04:
-  extends:
-    - .test:toolkit
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-containerd-ubuntu20.04:
-  extends:
-    - .test:containerd
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-crio-ubuntu20.04:
-  extends:
-    - .test:crio
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04
-
-test-docker-ubuntu20.04:
-  extends:
-    - .test:docker
-    - .dist-ubuntu20.04
-  needs:
-    - image-ubuntu20.04

.nvidia-ci.yml

@@ -39,19 +39,62 @@ variables:
   KITMAKER_RELEASE_FOLDER: "kitmaker"
   PACKAGE_ARCHIVE_RELEASE_FOLDER: "releases"
 
-.image-pull:
+# .copy-images copies the required application and packaging images from the
-  stage: image-build
+# IN_IMAGE="${IN_IMAGE_NAME}:${IN_IMAGE_TAG}${TAG_SUFFIX}"
+# to
+# OUT_IMAGE="${OUT_IMAGE_NAME}:${OUT_IMAGE_TAG}${TAG_SUFFIX}"
+# The script also logs into IN_REGISTRY and OUT_REGISTRY using the supplied
+# username and tokens.
+.copy-images:
+  parallel:
+    matrix:
+      - TAG_SUFFIX: ["", "-packaging"]
+  before_script:
+    - !reference [.regctl-setup, before_script]
+    - apk add --no-cache make bash
+  variables:
+    REGCTL: regctl
+  script:
+    - |
+      if [ -n ${IN_REGISTRY} ] && [ -n ${IN_REGISTRY_USER} ]; then
+        echo "Logging in to ${IN_REGISTRY}"
+        ${REGCTL} registry login "${IN_REGISTRY}" -u "${IN_REGISTRY_USER}" -p "${IN_REGISTRY_TOKEN}" || exit 1
+      fi
+
+      if [ -n ${OUT_REGISTRY} ] && [ -n ${OUT_REGISTRY_USER} ] && [ "${IN_REGISTRY}" != "${OUT_REGISTRY}" ]; then
+        echo "Logging in to ${OUT_REGISTRY}"
+        ${REGCTL} registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}" || exit 1
+      fi
+
+      export IN_IMAGE="${IN_IMAGE_NAME}:${IN_IMAGE_TAG}${TAG_SUFFIX}"
+      export OUT_IMAGE="${OUT_IMAGE_NAME}:${OUT_IMAGE_TAG}${TAG_SUFFIX}"
+
+      echo "Copying ${IN_IMAGE} to ${OUT_IMAGE}"
+      ${REGCTL} image copy ${IN_IMAGE} ${OUT_IMAGE}
+
+# pull-images pulls images from the public CI registry to the internal CI registry.
+pull-images:
+  extends:
+    - .copy-images
+  stage: pull
   variables:
     IN_REGISTRY: "${STAGING_REGISTRY}"
-    IN_IMAGE_NAME: container-toolkit
+    IN_IMAGE_NAME: ${STAGING_REGISTRY}/container-toolkit
-    IN_VERSION: "${STAGING_VERSION}"
+    IN_IMAGE_TAG: "${STAGING_VERSION}"
+
+    OUT_REGISTRY: "${CI_REGISTRY}"
     OUT_REGISTRY_USER: "${CI_REGISTRY_USER}"
     OUT_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
-    OUT_REGISTRY: "${CI_REGISTRY}"
     OUT_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
-    PUSH_MULTIPLE_TAGS: "false"
+    OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
   # We delay the job start to allow the public pipeline to generate the required images.
   rules:
+    # If the pipeline is triggered from a tag or the WEB UI we don't delay the
+    # start of the pipeline.
+    - if: $CI_COMMIT_TAG || $CI_PIPELINE_SOURCE == "web"
+    # If the pipeline is triggered through other means (i.e. a branch or MR)
+    # we add a 30 minute delay to ensure that the images are available in the
+    # public CI registry.
     - when: delayed
       start_in: 30 minutes
   timeout: 30 minutes
@@ -60,30 +103,6 @@ variables:
     when:
       - job_execution_timeout
       - stuck_or_timeout_failure
-  before_script:
-    - !reference [.regctl-setup, before_script]
-    - apk add --no-cache make bash
-    - >
-      regctl manifest get ${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} --list > /dev/null && echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST}" || ( echo "${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} does not exist" && sleep infinity )
-  script:
-    - regctl registry login "${OUT_REGISTRY}" -u "${OUT_REGISTRY_USER}" -p "${OUT_REGISTRY_TOKEN}"
-    - make -f deployments/container/Makefile IMAGE=${IN_REGISTRY}/${IN_IMAGE_NAME}:${IN_VERSION}-${DIST} OUT_IMAGE=${OUT_IMAGE_NAME}:${CI_COMMIT_SHORT_SHA}-${DIST} push-${DIST}
-
-image-ubi8:
-  extends:
-    - .dist-ubi8
-    - .image-pull
-
-image-ubuntu20.04:
-  extends:
-    - .dist-ubuntu20.04
-    - .image-pull
-
-# The DIST=packaging target creates an image containing all built packages
-image-packaging:
-  extends:
-    - .dist-packaging
-    - .image-pull
 
 # We skip the integration tests for the internal CI:
 .integration:
@@ -95,27 +114,37 @@ image-packaging:
 
 # The .scan step forms the base of the image scan operation performed before releasing
 # images.
-.scan:
+scan-images:
   stage: scan
+  needs:
+  - pull-images
   image: "${PULSE_IMAGE}"
+  parallel:
+    matrix:
+      - TAG_SUFFIX: [""]
+        PLATFORM: ["linux/amd64", "linux/arm64"]
+      - TAG_SUFFIX: "-packaging"
+        PLATFORM: "linux/amd64"
   variables:
-    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}-${DIST}"
+    IMAGE: "${CI_REGISTRY_IMAGE}/container-toolkit:${CI_COMMIT_SHORT_SHA}"
-    IMAGE_ARCHIVE: "container-toolkit-${DIST}-${ARCH}-${CI_JOB_ID}.tar"
+    IMAGE_ARCHIVE: "container-toolkit-${CI_JOB_ID}.tar"
   rules:
-    - if: $SKIP_SCANS != "yes"
+    - if: $IGNORE_SCANS == "yes"
-    - when: manual
+      allow_failure: true
-  before_script:
+    - when: on_success
-    - docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
-    # TODO: We should specify the architecture here and scan all architectures
-    - docker pull --platform="${PLATFORM}" "${IMAGE}"
-    - docker save "${IMAGE}" -o "${IMAGE_ARCHIVE}"
-    - AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
-    - >
-      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
-    - if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
   script:
-    - pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+    - |
-    - rm -f "${IMAGE_ARCHIVE}"
+      docker login -u "${CI_REGISTRY_USER}" -p "${CI_REGISTRY_PASSWORD}" "${CI_REGISTRY}"
+      export SCAN_IMAGE=${IMAGE}${TAG_SUFFIX}
+      echo "Scanning image ${SCAN_IMAGE} for ${PLATFORM}"
+      docker pull --platform="${PLATFORM}" "${SCAN_IMAGE}"
+      docker save "${SCAN_IMAGE}" -o "${IMAGE_ARCHIVE}"
+      AuthHeader=$(echo -n $SSA_CLIENT_ID:$SSA_CLIENT_SECRET | base64 -w0)
+      export SSA_TOKEN=$(curl --request POST --header "Authorization: Basic $AuthHeader" --header "Content-Type: application/x-www-form-urlencoded" ${SSA_ISSUER_URL} | jq ".access_token" |  tr -d '"')
+      if [ -z "$SSA_TOKEN" ]; then exit 1; else echo "SSA_TOKEN set!"; fi
+
+      pulse-cli -n $NSPECT_ID --ssa $SSA_TOKEN scan -i $IMAGE_ARCHIVE -p $CONTAINER_POLICY -o
+      rm -f "${IMAGE_ARCHIVE}"
   artifacts:
     when: always
     expire_in: 1 week
@@ -126,62 +155,10 @@ image-packaging:
       - vulns.json
       - policy_evaluation.json
 
-# Define the scan targets
+upload-kitmaker-packages:
-scan-ubuntu20.04-amd64:
-  extends:
-    - .dist-ubuntu20.04
-    - .platform-amd64
-    - .scan
-  needs:
-    - image-ubuntu20.04
-
-scan-ubuntu20.04-arm64:
-  extends:
-    - .dist-ubuntu20.04
-    - .platform-arm64
-    - .scan
-  needs:
-    - image-ubuntu20.04
-    - scan-ubuntu20.04-amd64
-
-scan-ubi8-amd64:
-  extends:
-    - .dist-ubi8
-    - .platform-amd64
-    - .scan
-  needs:
-    - image-ubi8
-
-scan-ubi8-arm64:
-  extends:
-    - .dist-ubi8
-    - .platform-arm64
-    - .scan
-  needs:
-    - image-ubi8
-    - scan-ubi8-amd64
-
-scan-packaging:
-  extends:
-    - .dist-packaging
-    - .scan
-  needs:
-    - image-packaging
-
-# Define external release helpers
-.release:ngc:
-  extends:
-    - .release:external
-  variables:
-    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
-    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
-    OUT_REGISTRY: "${NGC_REGISTRY}"
-    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
-
-.release:packages:
   stage: release
   needs:
-    - image-packaging
+    - pull-images
   variables:
     VERSION: "${CI_COMMIT_SHORT_SHA}"
     PACKAGE_REGISTRY: "${CI_REGISTRY}"
@@ -199,34 +176,81 @@ scan-packaging:
     - ./scripts/release-kitmaker-artifactory.sh "${KITMAKER_ARTIFACTORY_REPO}"
     - rm -rf ${ARTIFACTS_DIR}
 
-# Define the package release targets
+push-images-to-staging:
-release:packages:kitmaker:
   extends:
-    - .release:packages
+    - .copy-images
-
+  stage: release
-release:staging-ubuntu20.04:
-  extends:
-    - .release:staging
-    - .dist-ubuntu20.04
   needs:
-    - image-ubuntu20.04
+    - scan-images
+  variables:
+    IN_REGISTRY: "${CI_REGISTRY}"
+    IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
 
-# Define the external release targets
+    OUT_REGISTRY: "${NGC_REGISTRY}"
-# Release to NGC
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
-release:ngc-ubuntu20.04:
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
-  extends:
+    OUT_IMAGE_NAME: "${NGC_STAGING_REGISTRY}/container-toolkit"
-    - .dist-ubuntu20.04
+    OUT_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
-    - .release:ngc
 
-release:ngc-ubi8:
+.release-images:
   extends:
-    - .dist-ubi8
+    - .copy-images
-    - .release:ngc
+  stage: release
+  needs:
+    - scan-images
+    - push-images-to-staging
+  variables:
+    IN_REGISTRY: "${CI_REGISTRY}"
+    IN_REGISTRY_USER: "${CI_REGISTRY_USER}"
+    IN_REGISTRY_TOKEN: "${CI_REGISTRY_PASSWORD}"
+    IN_IMAGE_NAME: "${CI_REGISTRY_IMAGE}/container-toolkit"
+    IN_IMAGE_TAG: "${CI_COMMIT_SHORT_SHA}"
 
-release:ngc-packaging:
+    OUT_REGISTRY: "${NGC_REGISTRY}"
+    OUT_REGISTRY_USER: "${NGC_REGISTRY_USER}"
+    OUT_REGISTRY_TOKEN: "${NGC_REGISTRY_TOKEN}"
+    OUT_IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    OUT_IMAGE_TAG: "${CI_COMMIT_TAG}"
+
+release-images-to-ngc:
   extends:
-    - .dist-packaging
+    - .release-images
-    - .release:ngc
+  rules:
+    - if: $CI_COMMIT_TAG
+
+release-images-dummy:
+  extends:
+    - .release-images
+  variables:
+    REGCTL: "echo [DUMMY] regctl"
+  rules:
+    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
+
+# .sign-images forms the base of the jobs which sign images in the NGC registry.
+.sign-images:
+  stage: sign
+  image: ubuntu:latest
+  parallel:
+    matrix:
+      - TAG_SUFFIX: ["", "-packaging"]
+  variables:
+    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
+    IMAGE_TAG: "${CI_COMMIT_TAG}"
+    NGC_CLI: "ngc-cli/ngc"
+  before_script:
+    - !reference [.ngccli-setup, before_script]
+  script:
+    - |
+      # We ensure that the IMAGE_NAME and IMAGE_TAG is set
+      echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1
+      echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1
+
+      export IMAGE=${IMAGE_NAME}:${IMAGE_TAG}${TAG_SUFFIX}
+      echo "Signing the image ${IMAGE}"
+      ${NGC_CLI} registry image publish --source ${IMAGE} ${IMAGE} --public --discoverable --allow-guest --sign --org nvidia
 
 # Define the external image signing steps for NGC
 # Download the ngc cli binary for use in the sign steps
@@ -244,45 +268,24 @@ release:ngc-packaging:
     - unzip ngccli_linux.zip
     - chmod u+x ngc-cli/ngc
 
-# .sign forms the base of the deployment jobs which signs images in the CI registry.
+sign-ngc-images:
-# This is extended with the image name and version to be deployed.
+  extends:
-.sign:ngc:
+    - .sign-images
-  image: ubuntu:latest
+  needs:
-  stage: sign
+    - release-images-to-ngc
   rules:
     - if: $CI_COMMIT_TAG
   variables:
     NGC_CLI_API_KEY: "${NGC_REGISTRY_TOKEN}"
-    IMAGE_NAME: "${NGC_REGISTRY_IMAGE}"
-    IMAGE_TAG: "${CI_COMMIT_TAG}-${DIST}"
   retry:
     max: 2
-  before_script:
-    - !reference [.ngccli-setup, before_script]
-    # We ensure that the IMAGE_NAME and IMAGE_TAG is set
-    - 'echo Image Name: ${IMAGE_NAME} && [[ -n "${IMAGE_NAME}" ]] || exit 1'
-    - 'echo Image Tag: ${IMAGE_TAG} && [[ -n "${IMAGE_TAG}" ]] || exit 1'
-  script:
-    - 'echo "Signing the image ${IMAGE_NAME}:${IMAGE_TAG}"'
-    - ngc-cli/ngc registry image publish --source ${IMAGE_NAME}:${IMAGE_TAG} ${IMAGE_NAME}:${IMAGE_TAG} --public --discoverable --allow-guest --sign --org nvidia
 
-sign:ngc-ubuntu20.04:
+sign-images-dummy:
   extends:
-    - .dist-ubuntu20.04
+    - .sign-images
-    - .sign:ngc
   needs:
-    - release:ngc-ubuntu20.04
+    - release-images-dummy
-
+  variables:
-sign:ngc-ubi8:
+    NGC_CLI: "echo [DUMMY] ngc-cli/ngc"
-  extends:
+  rules:
-    - .dist-ubi8
+    - if: $CI_COMMIT_TAG == null || $CI_COMMIT_TAG == ""
-    - .sign:ngc
-  needs:
-    - release:ngc-ubi8
-
-sign:ngc-packaging:
-  extends:
-    - .dist-packaging
-    - .sign:ngc
-  needs:
-    - release:ngc-packaging

cmd/nvidia-ctk-installer/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-ctk-installer/toolkit"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/info"
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
 )
 
 const (
@@ -36,10 +37,11 @@ var signalReceived = make(chan bool, 1)
 type options struct {
 	toolkitInstallDir string
 
-	noDaemon   bool
+	noDaemon    bool
-	runtime    string
+	runtime     string
-	pidFile    string
+	pidFile     string
-	sourceRoot string
+	sourceRoot  string
+	packageType string
 
 	toolkitOptions toolkit.Options
 	runtimeOptions runtime.Options
@@ -123,11 +125,17 @@ func (a app) build() *cli.App {
 			EnvVars:     []string{"TOOLKIT_INSTALL_DIR", "ROOT"},
 		},
 		&cli.StringFlag{
-			Name:        "source-root",
+			Name:        "toolkit-source-root",
-			Value:       "/",
+			Usage:       "The folder where the required toolkit artifacts can be found. If this is not specified, the path /artifacts/{{ .ToolkitPackageType }} is used where ToolkitPackageType is the resolved package type",
-			Usage:       "The folder where the required toolkit artifacts can be found",
 			Destination: &options.sourceRoot,
-			EnvVars:     []string{"SOURCE_ROOT"},
+			EnvVars:     []string{"TOOLKIT_SOURCE_ROOT"},
+		},
+		&cli.StringFlag{
+			Name:        "toolkit-package-type",
+			Usage:       "specify the package type to use for the toolkit. One of ['deb', 'rpm', 'auto', '']. If 'auto' or '' are used, the type is inferred automatically.",
+			Value:       "auto",
+			Destination: &options.packageType,
+			EnvVars:     []string{"TOOLKIT_PACKAGE_TYPE"},
 		},
 		&cli.StringFlag{
 			Name:        "pid-file",
@@ -145,6 +153,15 @@ func (a app) build() *cli.App {
 }
 
 func (a *app) Before(c *cli.Context, o *options) error {
+	if o.sourceRoot == "" {
+		sourceRoot, err := a.resolveSourceRoot(o.runtimeOptions.HostRootMount, o.packageType)
+		if err != nil {
+			return fmt.Errorf("failed to resolve source root: %v", err)
+		}
+		a.logger.Infof("Resolved source root to %v", sourceRoot)
+		o.sourceRoot = sourceRoot
+	}
+
 	a.toolkit = toolkit.NewInstaller(
 		toolkit.WithLogger(a.logger),
 		toolkit.WithSourceRoot(o.sourceRoot),
@@ -277,3 +294,35 @@ func (a *app) shutdown(pidFile string) {
 		a.logger.Warningf("Unable to remove pidfile: %v", err)
 	}
 }
+
+func (a *app) resolveSourceRoot(hostRoot string, packageType string) (string, error) {
+	resolvedPackageType, err := a.resolvePackageType(hostRoot, packageType)
+	if err != nil {
+		return "", err
+	}
+	switch resolvedPackageType {
+	case "deb":
+		return "/artifacts/deb", nil
+	case "rpm":
+		return "/artifacts/rpm", nil
+	default:
+		return "", fmt.Errorf("invalid package type: %v", resolvedPackageType)
+	}
+}
+
+func (a *app) resolvePackageType(hostRoot string, packageType string) (rPackageTypes string, rerr error) {
+	if packageType != "" && packageType != "auto" {
+		return packageType, nil
+	}
+
+	locator := lookup.NewExecutableLocator(a.logger, hostRoot)
+	if candidates, err := locator.Locate("/usr/bin/rpm"); err == nil && len(candidates) > 0 {
+		return "rpm", nil
+	}
+
+	if candidates, err := locator.Locate("/usr/bin/dpkg"); err == nil && len(candidates) > 0 {
+		return "deb", nil
+	}
+
+	return "deb", nil
+}

cmd/nvidia-ctk-installer/main_test.go

@@ -433,7 +433,7 @@ swarm-resource = ""
 				"--driver-root-ctr-path=" + hostRoot,
 				"--pid-file=" + filepath.Join(testRoot, "toolkit.pid"),
 				"--restart-mode=none",
-				"--source-root=" + filepath.Join(artifactRoot, "deb"),
+				"--toolkit-source-root=" + filepath.Join(artifactRoot, "deb"),
 			}
 
 			err := app.Run(append(testArgs, tc.args...))

cmd/nvidia-ctk-installer/toolkit/installer/installer.go

@@ -47,7 +47,9 @@ var _ Installer = (*toolkitInstaller)(nil)
 
 // New creates a toolkit installer with the specified options.
 func New(opts ...Option) (Installer, error) {
-	t := &toolkitInstaller{}
+	t := &toolkitInstaller{
+		sourceRoot: "/",
+	}
 	for _, opt := range opts {
 		opt(t)
 	}
@@ -55,9 +57,6 @@ func New(opts ...Option) (Installer, error) {
 	if t.logger == nil {
 		t.logger = logger.New()
 	}
-	if t.sourceRoot == "" {
-		t.sourceRoot = "/"
-	}
 	if t.artifactRoot == nil {
 		artifactRoot, err := newArtifactRoot(t.logger, t.sourceRoot)
 		if err != nil {

cmd/nvidia-ctk-installer/toolkit/toolkit.go

@@ -215,7 +215,8 @@ func Flags(opts *Options) []cli.Flag {
 
 // An Installer is used to install the NVIDIA Container Toolkit from the toolkit container.
 type Installer struct {
-	logger     logger.Interface
+	logger logger.Interface
+
 	sourceRoot string
 	// toolkitRoot specifies the destination path at which the toolkit is installed.
 	toolkitRoot string

deployments/container/Dockerfile

@@ -0,0 +1,149 @@
+# SPDX-FileCopyrightText: Copyright (c) 2019 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+ARG GOLANG_VERSION=x.x.x
+ARG VERSION="N/A"
+
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 AS build
+
+RUN dnf install -y \
+    wget make git gcc \
+     && \
+    rm -rf /var/cache/yum/*
+
+ARG GOLANG_VERSION=x.x.x
+RUN set -eux; \
+    \
+    arch="$(uname -m)"; \
+    case "${arch##*-}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64 | arm64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
+    | tar -C /usr/local -xz
+
+
+ENV GOPATH=/go
+ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
+
+WORKDIR /build
+COPY . .
+
+RUN mkdir -p /artifacts/bin
+ARG VERSION="N/A"
+ARG GIT_COMMIT="unknown"
+RUN make PREFIX=/artifacts/bin cmd-nvidia-ctk-installer
+
+# The packaging stage collects the deb and rpm packages built for supported
+# architectures.
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 AS packaging
+
+ARG ARTIFACTS_ROOT
+COPY ${ARTIFACTS_ROOT} /artifacts/packages/
+
+WORKDIR /artifacts/packages
+
+# build-args are added to the manifest.txt file below.
+ARG PACKAGE_VERSION
+ARG GIT_BRANCH
+ARG GIT_COMMIT
+ARG GIT_COMMIT_SHORT
+ARG SOURCE_DATE_EPOCH
+ARG VERSION
+
+# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
+RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
+    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
+    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+
+# The debpackages stage is used to extract the contents of deb packages.
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubuntu20.04 AS debpackages
+
+ARG TARGETARCH
+ARG PACKAGE_DIST_DEB=ubuntu18.04
+
+COPY --from=packaging /artifacts/packages/${PACKAGE_DIST_DEB} /deb-packages
+
+RUN mkdir -p /artifacts/deb
+RUN set -eux; \
+    \
+    case "${TARGETARCH}" in \
+        x86_64 | amd64) ARCH='amd64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64 | arm64) ARCH='arm64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    for p in $(ls /deb-packages/${ARCH}/*.deb); do dpkg-deb -xv $p /artifacts/deb/; done
+
+# The rpmpackages stage is used to extract the contents of the rpm packages.
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 AS rpmpackages
+RUN dnf install -y cpio
+
+ARG TARGETARCH
+ARG PACKAGE_DIST_RPM=centos7
+
+COPY --from=packaging /artifacts/packages/${PACKAGE_DIST_RPM} /rpm-packages
+
+RUN mkdir -p /artifacts/rpm
+RUN set -eux; \
+    \
+    case "${TARGETARCH}" in \
+        x86_64 | amd64) ARCH='x86_64' ;; \
+        ppc64el | ppc64le) ARCH='ppc64le' ;; \
+        aarch64 | arm64) ARCH='aarch64' ;; \
+        *) echo "unsupported architecture" ; exit 1 ;; \
+    esac; \
+    for p in $(ls /rpm-packages/${ARCH}/*.rpm); do rpm2cpio $p | cpio -idmv -D /artifacts/rpm; done
+
+# The artifacts image serves as an intermediate stage to collect the artifacts
+# From the previous stages:
+# - The extracted deb packages
+# - The extracted rpm packages
+# - The nvidia-ctk-installer binary
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9 AS artifacts
+
+COPY --from=rpmpackages /artifacts/rpm /artifacts/rpm
+COPY --from=debpackages /artifacts/deb /artifacts/deb
+COPY --from=build /artifacts/bin /artifacts/build
+
+FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi9
+
+ENV NVIDIA_DISABLE_REQUIRE="true"
+ENV NVIDIA_VISIBLE_DEVICES=void
+ENV NVIDIA_DRIVER_CAPABILITIES=utility
+
+COPY --from=artifacts /artifacts/rpm /artifacts/rpm
+COPY --from=artifacts /artifacts/deb /artifacts/deb
+COPY --from=artifacts /artifacts/build /work
+
+WORKDIR /work
+ENV PATH=/work:$PATH
+
+ARG VERSION
+LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
+LABEL name="NVIDIA Container Runtime Config"
+LABEL vendor="NVIDIA"
+LABEL version="${VERSION}"
+LABEL release="N/A"
+LABEL summary="Automatically Configure your Container Runtime for GPU support."
+LABEL description="See summary"
+
+RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
+
+ENTRYPOINT ["/work/nvidia-ctk-installer"]

deployments/container/Dockerfile.packaging

@@ -1,38 +0,0 @@
-# Copyright (c) 2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG GOLANG_VERSION=x.x.x
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubuntu20.04
-
-ARG ARTIFACTS_ROOT
-COPY ${ARTIFACTS_ROOT} /artifacts/packages/
-
-WORKDIR /artifacts/packages
-
-# build-args are added to the manifest.txt file below.
-ARG PACKAGE_DIST
-ARG PACKAGE_VERSION
-ARG GIT_BRANCH
-ARG GIT_COMMIT
-ARG GIT_COMMIT_SHORT
-ARG SOURCE_DATE_EPOCH
-ARG VERSION
-
-# Create a manifest.txt file with the absolute paths of all deb and rpm packages in the container
-RUN echo "#IMAGE_EPOCH=$(date '+%s')" > /artifacts/manifest.txt && \
-    env | sed 's/^/#/g' >> /artifacts/manifest.txt && \
-    find /artifacts/packages -iname '*.deb' -o -iname '*.rpm' >> /artifacts/manifest.txt
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE

deployments/container/Dockerfile.ubi8

@@ -1,90 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi8 AS build
-
-RUN yum install -y \
-    wget make git gcc \
-     && \
-    rm -rf /var/cache/yum/*
-
-ARG GOLANG_VERSION=x.x.x
-RUN set -eux; \
-    \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-        x86_64 | amd64) ARCH='amd64' ;; \
-        ppc64el | ppc64le) ARCH='ppc64le' ;; \
-        aarch64 | arm64) ARCH='arm64' ;; \
-        *) echo "unsupported architecture" ; exit 1 ;; \
-    esac; \
-    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
-    | tar -C /usr/local -xz
-
-
-ENV GOPATH=/go
-ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
-
-WORKDIR /build
-COPY . .
-
-RUN mkdir /artifacts
-ARG VERSION="N/A"
-ARG GIT_COMMIT="unknown"
-RUN make PREFIX=/artifacts cmd-nvidia-ctk-installer
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubi8
-
-ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=void
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
-
-ARG ARTIFACTS_ROOT
-ARG PACKAGE_DIST
-COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
-
-WORKDIR /artifacts/packages
-
-ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH=${TARGETARCH}
-
-RUN PACKAGE_ARCH=${PACKAGE_ARCH/amd64/x86_64} && PACKAGE_ARCH=${PACKAGE_ARCH/arm64/aarch64} && \
-    yum localinstall -y \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1-1.*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools-1.*.rpm \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*-${PACKAGE_VERSION}*.rpm
-
-WORKDIR /work
-
-COPY --from=build /artifacts/nvidia-ctk-installer /work/nvidia-ctk-installer
-RUN ln -s nvidia-ctk-installer nvidia-toolkit
-
-ENV PATH=/work:$PATH
-
-ARG VERSION
-LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
-LABEL name="NVIDIA Container Runtime Config"
-LABEL vendor="NVIDIA"
-LABEL version="${VERSION}"
-LABEL release="N/A"
-LABEL summary="Automatically Configure your Container Runtime for GPU support."
-LABEL description="See summary"
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
-
-ENTRYPOINT ["/work/nvidia-ctk-installer"]

deployments/container/Dockerfile.ubuntu

@@ -1,98 +0,0 @@
-# Copyright (c) 2019-2021, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-
-ARG GOLANG_VERSION=x.x.x
-ARG VERSION="N/A"
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubuntu20.04 AS build
-
-RUN apt-get update && \
-    apt-get install -y wget make git gcc \
-    && \
-    rm -rf /var/lib/apt/lists/*
-
-ARG GOLANG_VERSION=x.x.x
-RUN set -eux; \
-    \
-    arch="$(uname -m)"; \
-    case "${arch##*-}" in \
-        x86_64 | amd64) ARCH='amd64' ;; \
-        ppc64el | ppc64le) ARCH='ppc64le' ;; \
-        aarch64 | arm64) ARCH='arm64' ;; \
-        *) echo "unsupported architecture" ; exit 1 ;; \
-    esac; \
-    wget -nv -O - https://storage.googleapis.com/golang/go${GOLANG_VERSION}.linux-${ARCH}.tar.gz \
-    | tar -C /usr/local -xz
-
-ENV GOPATH=/go
-ENV PATH=$GOPATH/bin:/usr/local/go/bin:$PATH
-
-WORKDIR /build
-COPY . .
-
-RUN mkdir /artifacts
-ARG VERSION="N/A"
-ARG GIT_COMMIT="unknown"
-RUN make PREFIX=/artifacts cmd-nvidia-ctk-installer
-
-FROM nvcr.io/nvidia/cuda:12.9.0-base-ubuntu20.04
-
-# Remove the CUDA repository configurations to avoid issues with rotated GPG keys
-RUN rm -f /etc/apt/sources.list.d/cuda.list
-
-ARG DEBIAN_FRONTEND=noninteractive
-RUN apt-get update && apt-get install -y --no-install-recommends \
-    libcap2 \
-    curl \
-        && \
-    rm -rf /var/lib/apt/lists/*
-
-ENV NVIDIA_DISABLE_REQUIRE="true"
-ENV NVIDIA_VISIBLE_DEVICES=void
-ENV NVIDIA_DRIVER_CAPABILITIES=utility
-
-ARG ARTIFACTS_ROOT
-ARG PACKAGE_DIST
-COPY ${ARTIFACTS_ROOT}/${PACKAGE_DIST} /artifacts/packages/${PACKAGE_DIST}
-
-WORKDIR /artifacts/packages
-
-ARG PACKAGE_VERSION
-ARG TARGETARCH
-ENV PACKAGE_ARCH=${TARGETARCH}
-
-RUN dpkg -i \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container1_1.*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/libnvidia-container-tools_1.*.deb \
-    ${PACKAGE_DIST}/${PACKAGE_ARCH}/nvidia-container-toolkit*_${PACKAGE_VERSION}*.deb
-
-WORKDIR /work
-
-COPY --from=build /artifacts/nvidia-ctk-installer /work/nvidia-ctk-installer
-RUN ln -s nvidia-ctk-installer nvidia-toolkit
-
-ENV PATH=/work:$PATH
-
-ARG VERSION
-LABEL io.k8s.display-name="NVIDIA Container Runtime Config"
-LABEL name="NVIDIA Container Runtime Config"
-LABEL vendor="NVIDIA"
-LABEL version="${VERSION}"
-LABEL release="N/A"
-LABEL summary="Automatically Configure your Container Runtime for GPU support."
-LABEL description="See summary"
-
-RUN mkdir /licenses && mv /NGC-DL-CONTAINER-LICENSE /licenses/NGC-DL-CONTAINER-LICENSE
-
-ENTRYPOINT ["/work/nvidia-ctk-installer"]

deployments/container/Makefile

@@ -29,17 +29,17 @@ include $(CURDIR)/versions.mk
 
 IMAGE_VERSION := $(VERSION)
 
-IMAGE_TAG ?= $(VERSION)-$(DIST)
+IMAGE_TAG ?= $(VERSION)
 IMAGE = $(IMAGE_NAME):$(IMAGE_TAG)
 
 OUT_IMAGE_NAME ?= $(IMAGE_NAME)
 OUT_IMAGE_VERSION ?= $(IMAGE_VERSION)
-OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)-$(DIST)
+OUT_IMAGE_TAG = $(OUT_IMAGE_VERSION)
 OUT_IMAGE = $(OUT_IMAGE_NAME):$(OUT_IMAGE_TAG)
 
 ##### Public rules #####
-DEFAULT_PUSH_TARGET := ubuntu20.04
+DEFAULT_PUSH_TARGET := ubi9
-DISTRIBUTIONS := ubuntu20.04 ubi8
+DISTRIBUTIONS := $(DEFAULT_PUSH_TARGET)
 
 META_TARGETS := packaging
 
@@ -56,30 +56,16 @@ else
 include $(CURDIR)/deployments/container/multi-arch.mk
 endif
 
-# For the default push target we also push a short tag equal to the version.
-# We skip this for the development release
-DEVEL_RELEASE_IMAGE_VERSION ?= devel
-PUSH_MULTIPLE_TAGS ?= true
-ifeq ($(strip $(OUT_IMAGE_VERSION)),$(DEVEL_RELEASE_IMAGE_VERSION))
-PUSH_MULTIPLE_TAGS = false
-endif
-ifeq ($(PUSH_MULTIPLE_TAGS),true)
-push-$(DEFAULT_PUSH_TARGET): push-short
-endif
-
-push-%: DIST = $(*)
-push-short: DIST = $(DEFAULT_PUSH_TARGET)
-
 # Define the push targets
 $(PUSH_TARGETS): push-%:
 	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
 
-push-short:
+DOCKERFILE = $(CURDIR)/deployments/container/Dockerfile
-	$(CURDIR)/scripts/publish-image.sh $(IMAGE) $(OUT_IMAGE)
 
-
+# For packaging targets we set the output image tag to include the -packaging suffix.
-build-%: DIST = $(*)
+%-packaging: INTERMEDIATE_TARGET := --target=packaging
-build-%: DOCKERFILE = $(CURDIR)/deployments/container/Dockerfile.$(DOCKERFILE_SUFFIX)
+%-packaging: IMAGE_TAG = $(IMAGE_VERSION)-packaging
+%-packaging: OUT_IMAGE_TAG = $(IMAGE_VERSION)-packaging
 
 ARTIFACTS_ROOT ?= $(shell realpath --relative-to=$(CURDIR) $(DIST_DIR))
 
@@ -90,10 +76,12 @@ $(IMAGE_TARGETS): image-%: $(ARTIFACTS_ROOT)
 		--provenance=false --sbom=false \
 		$(DOCKER_BUILD_OPTIONS) \
 		$(DOCKER_BUILD_PLATFORM_OPTIONS) \
+		$(INTERMEDIATE_TARGET) \
 		--tag $(IMAGE) \
 		--build-arg ARTIFACTS_ROOT="$(ARTIFACTS_ROOT)" \
 		--build-arg GOLANG_VERSION="$(GOLANG_VERSION)" \
-		--build-arg PACKAGE_DIST="$(PACKAGE_DIST)" \
+		--build-arg PACKAGE_DIST_DEB="$(PACKAGE_DIST_DEB)" \
+		--build-arg PACKAGE_DIST_RPM="$(PACKAGE_DIST_RPM)" \
 		--build-arg PACKAGE_VERSION="$(PACKAGE_VERSION)" \
 		--build-arg VERSION="$(VERSION)" \
 		--build-arg GIT_COMMIT="$(GIT_COMMIT)" \
@@ -103,25 +91,19 @@ $(IMAGE_TARGETS): image-%: $(ARTIFACTS_ROOT)
 		-f $(DOCKERFILE) \
 		$(CURDIR)
 
-build-ubuntu%: DOCKERFILE_SUFFIX := ubuntu
-build-ubuntu%: PACKAGE_DIST = ubuntu18.04
 
-build-ubi8: DOCKERFILE_SUFFIX := ubi8
+PACKAGE_DIST_DEB = ubuntu18.04
-build-ubi8: PACKAGE_DIST = centos7
+# TODO: This needs to be set to centos8 for ppc64le builds
+PACKAGE_DIST_RPM = centos7
 
-build-packaging: DOCKERFILE_SUFFIX := packaging
+# Handle the default build target.
-build-packaging: PACKAGE_ARCH := amd64
+.PHONY: build push
-build-packaging: PACKAGE_DIST = all
+build: build-$(DEFAULT_PUSH_TARGET)
+push: push-$(DEFAULT_PUSH_TARGET)
 
 # Test targets
 test-%: DIST = $(*)
 
-# Handle the default build target.
-.PHONY: build
-build: $(DEFAULT_PUSH_TARGET)
-$(DEFAULT_PUSH_TARGET): build-$(DEFAULT_PUSH_TARGET)
-$(DEFAULT_PUSH_TARGET): DIST = $(DEFAULT_PUSH_TARGET)
-
 TEST_CASES ?= docker crio containerd
 $(TEST_TARGETS): test-%:
 	TEST_CASES="$(TEST_CASES)" bash -x $(CURDIR)/test/container/main.sh run \

deployments/container/native-only.mk

@@ -23,11 +23,3 @@ $(BUILD_TARGETS): build-%: image-%
 else
 $(BUILD_TARGETS): build-%: image-%
 endif
-
-# For the default distribution we also retag the image.
-# Note: This needs to be updated for multi-arch images.
-ifeq ($(IMAGE_TAG),$(VERSION)-$(DIST))
-$(DEFAULT_PUSH_TARGET):
-	$(DOCKER) image inspect $(IMAGE) > /dev/null || $(DOCKER) pull $(IMAGE)
-	$(DOCKER) tag $(IMAGE) $(subst :$(IMAGE_TAG),:$(VERSION),$(IMAGE))
-endif

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/google/uuid v1.6.0 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
+	github.com/opencontainers/cgroups v0.0.1 // indirect
 	github.com/opencontainers/runtime-tools v0.9.1-0.20221107090550-2e043c6bd626 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/rogpeppe/go-internal v1.11.0 // indirect

go.sum

@@ -37,6 +37,8 @@ github.com/moby/sys/reexec v0.1.0/go.mod h1:EqjBg8F3X7iZe5pU6nRZnYCMUTXoxsjiIfHu
 github.com/moby/sys/symlink v0.3.0 h1:GZX89mEZ9u53f97npBy4Rc3vJKj7JBDj/PN2I22GrNU=
 github.com/moby/sys/symlink v0.3.0/go.mod h1:3eNdhduHmYPcgsJtZXW1W4XUJdZGBIkttZ8xKqPUJq0=
 github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ=
+github.com/opencontainers/cgroups v0.0.1 h1:MXjMkkFpKv6kpuirUa4USFBas573sSAY082B4CiHEVA=
+github.com/opencontainers/cgroups v0.0.1/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs=
 github.com/opencontainers/runc v1.3.0 h1:cvP7xbEvD0QQAs0nZKLzkVog2OPZhI/V2w3WmTmUSXI=
 github.com/opencontainers/runc v1.3.0/go.mod h1:9wbWt42gV+KRxKRVVugNP6D5+PQciRbenB4fLVsqGPs=
 github.com/opencontainers/runtime-spec v1.0.3-0.20220825212826-86290f6a00fb/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=

internal/config/image/cuda_image.go

@@ -19,6 +19,7 @@ package image
 import (
 	"fmt"
 	"path/filepath"
+	"slices"
 	"strconv"
 	"strings"
 
@@ -143,8 +144,8 @@ func (i CUDA) HasDisableRequire() bool {
 	return false
 }
 
-// DevicesFromEnvvars returns the devices requested by the image through environment variables
+// devicesFromEnvvars returns the devices requested by the image through environment variables
-func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
+func (i CUDA) devicesFromEnvvars(envVars ...string) []string {
 	// We concantenate all the devices from the specified env.
 	var isSet bool
 	var devices []string
@@ -165,15 +166,15 @@ func (i CUDA) DevicesFromEnvvars(envVars ...string) VisibleDevices {
 
 	// Environment variable unset with legacy image: default to "all".
 	if !isSet && len(devices) == 0 && i.IsLegacy() {
-		return NewVisibleDevices("all")
+		devices = []string{"all"}
 	}
 
 	// Environment variable unset or empty or "void": return nil
 	if len(devices) == 0 || requested["void"] {
-		return NewVisibleDevices("void")
+		devices = []string{"void"}
 	}
 
-	return NewVisibleDevices(devices...)
+	return NewVisibleDevices(devices...).List()
 }
 
 // GetDriverCapabilities returns the requested driver capabilities.
@@ -232,6 +233,22 @@ func (i CUDA) OnlyFullyQualifiedCDIDevices() bool {
 	return hasCDIdevice
 }
 
+// visibleEnvVars returns the environment variables that are used to determine device visibility.
+// It returns the preferred environment variables that are set, or NVIDIA_VISIBLE_DEVICES if none are set.
+func (i CUDA) visibleEnvVars() []string {
+	var envVars []string
+	for _, envVar := range i.preferredVisibleDeviceEnvVars {
+		if !i.HasEnvvar(envVar) {
+			continue
+		}
+		envVars = append(envVars, envVar)
+	}
+	if len(envVars) > 0 {
+		return envVars
+	}
+	return []string{EnvVarNvidiaVisibleDevices}
+}
+
 // VisibleDevices returns a list of devices requested in the container image.
 // If volume mount requests are enabled these are returned if requested,
 // otherwise device requests through environment variables are considered.
@@ -253,7 +270,7 @@ func (i CUDA) VisibleDevices() []string {
 	}
 
 	// Get the Fallback to reading from the environment variable if privileges are correct
-	envVarDeviceRequests := i.VisibleDevicesFromEnvVar()
+	envVarDeviceRequests := i.visibleDevicesFromEnvVar()
 	if len(envVarDeviceRequests) == 0 {
 		return nil
 	}
@@ -265,7 +282,10 @@ func (i CUDA) VisibleDevices() []string {
 	}
 
 	// We log a warning if we are ignoring the environment variable requests.
-	i.logger.Warningf("Ignoring devices specified in NVIDIA_VISIBLE_DEVICES in unprivileged container")
+	envVars := i.visibleEnvVars()
+	if len(envVars) > 0 {
+		i.logger.Warningf("Ignoring devices requested by environment variable(s) in unprivileged container: %v", envVars)
+	}
 
 	return nil
 }
@@ -281,31 +301,34 @@ func (i CUDA) cdiDeviceRequestsFromAnnotations() []string {
 		return nil
 	}
 
-	var devices []string
+	var annotationKeys []string
-	for key, value := range i.annotations {
+	for key := range i.annotations {
 		for _, prefix := range i.annotationsPrefixes {
 			if strings.HasPrefix(key, prefix) {
-				devices = append(devices, strings.Split(value, ",")...)
+				annotationKeys = append(annotationKeys, key)
 				// There is no need to check additional prefixes since we
 				// typically deduplicate devices in any case.
 				break
 			}
 		}
 	}
+	// We sort the annotationKeys for consistent results.
+	slices.Sort(annotationKeys)
+
+	var devices []string
+	for _, key := range annotationKeys {
+		devices = append(devices, strings.Split(i.annotations[key], ",")...)
+	}
 	return devices
 }
 
-// VisibleDevicesFromEnvVar returns the set of visible devices requested through environment variables.
+// visibleDevicesFromEnvVar returns the set of visible devices requested through environment variables.
 // If any of the preferredVisibleDeviceEnvVars are present in the image, they
 // are used to determine the visible devices. If this is not the case, the
 // NVIDIA_VISIBLE_DEVICES environment variable is used.
-func (i CUDA) VisibleDevicesFromEnvVar() []string {
+func (i CUDA) visibleDevicesFromEnvVar() []string {
-	for _, envVar := range i.preferredVisibleDeviceEnvVars {
+	envVars := i.visibleEnvVars()
-		if i.HasEnvvar(envVar) {
+	return i.devicesFromEnvvars(envVars...)
-			return i.DevicesFromEnvvars(i.preferredVisibleDeviceEnvVars...).List()
-		}
-	}
-	return i.DevicesFromEnvvars(EnvVarNvidiaVisibleDevices).List()
 }
 
 // visibleDevicesFromMounts returns the set of visible devices requested as mounts.
@@ -391,7 +414,7 @@ func (m cdiDeviceMountRequest) qualifiedName() (string, error) {
 
 // ImexChannelsFromEnvVar returns the list of IMEX channels requested for the image.
 func (i CUDA) ImexChannelsFromEnvVar() []string {
-	imexChannels := i.DevicesFromEnvvars(EnvVarNvidiaImexChannels).List()
+	imexChannels := i.devicesFromEnvvars(EnvVarNvidiaImexChannels)
 	if len(imexChannels) == 1 && imexChannels[0] == "all" {
 		return nil
 	}

internal/config/image/cuda_image_test.go

@@ -429,7 +429,7 @@ func TestGetDevicesFromEnvvar(t *testing.T) {
 			)
 
 			require.NoError(t, err)
-			devices := image.VisibleDevicesFromEnvVar()
+			devices := image.visibleDevicesFromEnvVar()
 			require.EqualValues(t, tc.expectedDevices, devices)
 		})
 	}
@@ -508,13 +508,15 @@ func TestGetVisibleDevicesFromMounts(t *testing.T) {
 
 func TestVisibleDevices(t *testing.T) {
 	var tests = []struct {
-		description        string
+		description                   string
-		mountDevices       []specs.Mount
+		mountDevices                  []specs.Mount
-		envvarDevices      string
+		envvarDevices                 string
-		privileged         bool
+		privileged                    bool
-		acceptUnprivileged bool
+		acceptUnprivileged            bool
-		acceptMounts       bool
+		acceptMounts                  bool
-		expectedDevices    []string
+		preferredVisibleDeviceEnvVars []string
+		env                           map[string]string
+		expectedDevices               []string
 	}{
 		{
 			description: "Mount devices, unprivileged, no accept unprivileged",
@@ -597,20 +599,92 @@ func TestVisibleDevices(t *testing.T) {
 			acceptMounts:       false,
 			expectedDevices:    nil,
 		},
+		// New test cases for visibleEnvVars functionality
+		{
+			description:                   "preferred env var set and present in env, privileged",
+			mountDevices:                  nil,
+			envvarDevices:                 "",
+			privileged:                    true,
+			acceptUnprivileged:            false,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS"},
+			env: map[string]string{
+				"DOCKER_RESOURCE_GPUS": "GPU-12345",
+			},
+			expectedDevices: []string{"GPU-12345"},
+		},
+		{
+			description:                   "preferred env var set and present in env, unprivileged but accepted",
+			mountDevices:                  nil,
+			envvarDevices:                 "",
+			privileged:                    false,
+			acceptUnprivileged:            true,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS"},
+			env: map[string]string{
+				"DOCKER_RESOURCE_GPUS": "GPU-12345",
+			},
+			expectedDevices: []string{"GPU-12345"},
+		},
+		{
+			description:                   "preferred env var set and present in env, unprivileged and not accepted",
+			mountDevices:                  nil,
+			envvarDevices:                 "",
+			privileged:                    false,
+			acceptUnprivileged:            false,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS"},
+			env: map[string]string{
+				"DOCKER_RESOURCE_GPUS": "GPU-12345",
+			},
+			expectedDevices: nil,
+		},
+		{
+			description:                   "multiple preferred env vars, both present, privileged",
+			mountDevices:                  nil,
+			envvarDevices:                 "",
+			privileged:                    true,
+			acceptUnprivileged:            false,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS", "DOCKER_RESOURCE_GPUS_ADDITIONAL"},
+			env: map[string]string{
+				"DOCKER_RESOURCE_GPUS":            "GPU-12345",
+				"DOCKER_RESOURCE_GPUS_ADDITIONAL": "GPU-67890",
+			},
+			expectedDevices: []string{"GPU-12345", "GPU-67890"},
+		},
+		{
+			description:                   "preferred env var not present, fallback to NVIDIA_VISIBLE_DEVICES, privileged",
+			mountDevices:                  nil,
+			envvarDevices:                 "GPU-12345",
+			privileged:                    true,
+			acceptUnprivileged:            false,
+			acceptMounts:                  true,
+			preferredVisibleDeviceEnvVars: []string{"DOCKER_RESOURCE_GPUS"},
+			env: map[string]string{
+				EnvVarNvidiaVisibleDevices: "GPU-12345",
+			},
+			expectedDevices: []string{"GPU-12345"},
+		},
 	}
 	for _, tc := range tests {
 		t.Run(tc.description, func(t *testing.T) {
-			// Wrap the call to getDevices() in a closure.
+			// Create env map with both NVIDIA_VISIBLE_DEVICES and any additional env vars
+			env := make(map[string]string)
+			if tc.envvarDevices != "" {
+				env[EnvVarNvidiaVisibleDevices] = tc.envvarDevices
+			}
+			for k, v := range tc.env {
+				env[k] = v
+			}
+
 			image, err := New(
-				WithEnvMap(
+				WithEnvMap(env),
-					map[string]string{
-						EnvVarNvidiaVisibleDevices: tc.envvarDevices,
-					},
-				),
 				WithMounts(tc.mountDevices),
 				WithPrivileged(tc.privileged),
 				WithAcceptDeviceListAsVolumeMounts(tc.acceptMounts),
 				WithAcceptEnvvarUnprivileged(tc.acceptUnprivileged),
+				WithPreferredVisibleDevicesEnvVars(tc.preferredVisibleDeviceEnvVars...),
 			)
 			require.NoError(t, err)
 			require.Equal(t, tc.expectedDevices, image.VisibleDevices())

internal/edits/device.go

@@ -20,6 +20,8 @@ import (
 	"tags.cncf.io/container-device-interface/pkg/cdi"
 	"tags.cncf.io/container-device-interface/specs-go"
 
+	"github.com/opencontainers/runc/libcontainer/devices"
+
 	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
 )
 
@@ -43,19 +45,37 @@ func (d device) toEdits() (*cdi.ContainerEdits, error) {
 // toSpec converts a discovered Device to a CDI Spec Device. Note
 // that missing info is filled in when edits are applied by querying the Device node.
 func (d device) toSpec() (*specs.DeviceNode, error) {
+	s := d.fromPathOrDefault()
 	// The HostPath field was added in the v0.5.0 CDI specification.
 	// The cdi package uses strict unmarshalling when loading specs from file causing failures for
 	// unexpected fields.
 	// Since the behaviour for HostPath == "" and HostPath == Path are equivalent, we clear HostPath
 	// if it is equal to Path to ensure compatibility with the widest range of specs.
-	hostPath := d.HostPath
+	if s.HostPath == d.Path {
-	if hostPath == d.Path {
+		s.HostPath = ""
-		hostPath = ""
-	}
-	s := specs.DeviceNode{
-		HostPath: hostPath,
-		Path:     d.Path,
 	}
 
-	return &s, nil
+	return s, nil
+}
+
+// fromPathOrDefault attempts to return the returns the information about the
+// CDI device from the specified host path.
+// If this fails a minimal device is returned so that this information can be
+// queried by the container runtime such as containerd.
+func (d device) fromPathOrDefault() *specs.DeviceNode {
+	dn, err := devices.DeviceFromPath(d.HostPath, "rwm")
+	if err != nil {
+		return &specs.DeviceNode{
+			HostPath: d.HostPath,
+			Path:     d.Path,
+		}
+	}
+
+	return &specs.DeviceNode{
+		HostPath: d.HostPath,
+		Path:     d.Path,
+		Major:    dn.Major,
+		Minor:    dn.Minor,
+		FileMode: &dn.FileMode,
+	}
 }

internal/modifier/cdi_test.go

@@ -98,7 +98,7 @@ func TestDeviceRequests(t *testing.T) {
 					"another-prefix/bar": "example.com/device=baz",
 				},
 			},
-			expectedDevices: []string{"example.com/device=bar", "example.com/device=baz"},
+			expectedDevices: []string{"example.com/device=baz", "example.com/device=bar"},
 		},
 		{
 			description: "multiple matching annotations with duplicate devices",

internal/modifier/csv.go

@@ -33,7 +33,7 @@ import (
 // NewCSVModifier creates a modifier that applies modications to an OCI spec if required by the runtime wrapper.
 // The modifications are defined by CSV MountSpecs.
 func NewCSVModifier(logger logger.Interface, cfg *config.Config, container image.CUDA) (oci.SpecModifier, error) {
-	if devices := container.VisibleDevicesFromEnvVar(); len(devices) == 0 {
+	if devices := container.VisibleDevices(); len(devices) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}

internal/modifier/gated.go

@@ -37,7 +37,7 @@ import (
 //
 // If not devices are selected, no changes are made.
 func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image image.CUDA, driver *root.Driver, hookCreator discover.HookCreator) (oci.SpecModifier, error) {
-	if devices := image.VisibleDevicesFromEnvVar(); len(devices) == 0 {
+	if devices := image.VisibleDevices(); len(devices) == 0 {
 		logger.Infof("No modification required; no devices requested")
 		return nil, nil
 	}

internal/modifier/graphics.go

@@ -29,9 +29,10 @@ import (
 
 // NewGraphicsModifier constructs a modifier that injects graphics-related modifications into an OCI runtime specification.
 // The value of the NVIDIA_DRIVER_CAPABILITIES environment variable is checked to determine if this modification should be made.
-func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, containerImage image.CUDA, driver *root.Driver, hookCreator discover.HookCreator) (oci.SpecModifier, error) {
+func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, container image.CUDA, driver *root.Driver, hookCreator discover.HookCreator) (oci.SpecModifier, error) {
-	if required, reason := requiresGraphicsModifier(containerImage); !required {
+	devices, reason := requiresGraphicsModifier(container)
-		logger.Infof("No graphics modifier required: %v", reason)
+	if len(devices) == 0 {
+		logger.Infof("No graphics modifier required; %v", reason)
 		return nil, nil
 	}
 
@@ -48,7 +49,7 @@ func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, containerI
 	devRoot := driver.Root
 	drmNodes, err := discover.NewDRMNodesDiscoverer(
 		logger,
-		containerImage.DevicesFromEnvvars(image.EnvVarNvidiaVisibleDevices),
+		image.NewVisibleDevices(devices...),
 		devRoot,
 		hookCreator,
 	)
@@ -64,14 +65,15 @@ func NewGraphicsModifier(logger logger.Interface, cfg *config.Config, containerI
 }
 
 // requiresGraphicsModifier determines whether a graphics modifier is required.
-func requiresGraphicsModifier(cudaImage image.CUDA) (bool, string) {
+func requiresGraphicsModifier(cudaImage image.CUDA) ([]string, string) {
-	if devices := cudaImage.VisibleDevicesFromEnvVar(); len(devices) == 0 {
+	devices := cudaImage.VisibleDevices()
-		return false, "no devices requested"
+	if len(devices) == 0 {
+		return nil, "no devices requested"
 	}
 
 	if !cudaImage.GetDriverCapabilities().Any(image.DriverCapabilityGraphics, image.DriverCapabilityDisplay) {
-		return false, "no required capabilities requested"
+		return nil, "no required capabilities requested"
 	}
 
-	return true, ""
+	return devices, ""
 }

internal/modifier/graphics_test.go

@@ -26,9 +26,9 @@ import (
 
 func TestGraphicsModifier(t *testing.T) {
 	testCases := []struct {
-		description      string
+		description     string
-		envmap           map[string]string
+		envmap          map[string]string
-		expectedRequired bool
+		expectedDevices []string
 	}{
 		{
 			description: "empty image does not create modifier",
@@ -52,7 +52,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "all",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 		{
 			description: "devices with graphics capability creates modifier",
@@ -60,7 +60,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "graphics",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 		{
 			description: "devices with compute,graphics capability creates modifier",
@@ -68,7 +68,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "compute,graphics",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 		{
 			description: "devices with display capability creates modifier",
@@ -76,7 +76,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "display",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 		{
 			description: "devices with display,graphics capability creates modifier",
@@ -84,7 +84,7 @@ func TestGraphicsModifier(t *testing.T) {
 				"NVIDIA_VISIBLE_DEVICES":     "all",
 				"NVIDIA_DRIVER_CAPABILITIES": "display,graphics",
 			},
-			expectedRequired: true,
+			expectedDevices: []string{"all"},
 		},
 	}
 
@@ -94,7 +94,7 @@ func TestGraphicsModifier(t *testing.T) {
 				image.WithEnvMap(tc.envmap),
 			)
 			required, _ := requiresGraphicsModifier(image)
-			require.EqualValues(t, tc.expectedRequired, required)
+			require.EqualValues(t, tc.expectedDevices, required)
 		})
 	}
 }

tests/e2e/Makefile

@@ -1,4 +1,4 @@
-# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 
+# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 # SPDX-License-Identifier: Apache-2.0
 #
 # Licensed under the Apache License, Version 2.0 (the "License");

vendor/github.com/opencontainers/cgroups/LICENSE

@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

vendor/github.com/opencontainers/cgroups/devices/config/device.go

@@ -0,0 +1,174 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"strconv"
+)
+
+const (
+	Wildcard = -1
+)
+
+type Device struct {
+	Rule
+
+	// Path to the device.
+	Path string `json:"path"`
+
+	// FileMode permission bits for the device.
+	FileMode os.FileMode `json:"file_mode"`
+
+	// Uid of the device.
+	Uid uint32 `json:"uid"`
+
+	// Gid of the device.
+	Gid uint32 `json:"gid"`
+}
+
+// Permissions is a cgroupv1-style string to represent device access. It
+// has to be a string for backward compatibility reasons, hence why it has
+// methods to do set operations.
+type Permissions string
+
+const (
+	deviceRead uint = (1 << iota)
+	deviceWrite
+	deviceMknod
+)
+
+func (p Permissions) toSet() uint {
+	var set uint
+	for _, perm := range p {
+		switch perm {
+		case 'r':
+			set |= deviceRead
+		case 'w':
+			set |= deviceWrite
+		case 'm':
+			set |= deviceMknod
+		}
+	}
+	return set
+}
+
+func fromSet(set uint) Permissions {
+	var perm string
+	if set&deviceRead == deviceRead {
+		perm += "r"
+	}
+	if set&deviceWrite == deviceWrite {
+		perm += "w"
+	}
+	if set&deviceMknod == deviceMknod {
+		perm += "m"
+	}
+	return Permissions(perm)
+}
+
+// Union returns the union of the two sets of Permissions.
+func (p Permissions) Union(o Permissions) Permissions {
+	lhs := p.toSet()
+	rhs := o.toSet()
+	return fromSet(lhs | rhs)
+}
+
+// Difference returns the set difference of the two sets of Permissions.
+// In set notation, A.Difference(B) gives you A\B.
+func (p Permissions) Difference(o Permissions) Permissions {
+	lhs := p.toSet()
+	rhs := o.toSet()
+	return fromSet(lhs &^ rhs)
+}
+
+// Intersection computes the intersection of the two sets of Permissions.
+func (p Permissions) Intersection(o Permissions) Permissions {
+	lhs := p.toSet()
+	rhs := o.toSet()
+	return fromSet(lhs & rhs)
+}
+
+// IsEmpty returns whether the set of permissions in a Permissions is
+// empty.
+func (p Permissions) IsEmpty() bool {
+	return p == Permissions("")
+}
+
+// IsValid returns whether the set of permissions is a subset of valid
+// permissions (namely, {r,w,m}).
+func (p Permissions) IsValid() bool {
+	return p == fromSet(p.toSet())
+}
+
+type Type rune
+
+const (
+	WildcardDevice Type = 'a'
+	BlockDevice    Type = 'b'
+	CharDevice     Type = 'c' // or 'u'
+	FifoDevice     Type = 'p'
+)
+
+func (t Type) IsValid() bool {
+	switch t {
+	case WildcardDevice, BlockDevice, CharDevice, FifoDevice:
+		return true
+	default:
+		return false
+	}
+}
+
+func (t Type) CanMknod() bool {
+	switch t {
+	case BlockDevice, CharDevice, FifoDevice:
+		return true
+	default:
+		return false
+	}
+}
+
+func (t Type) CanCgroup() bool {
+	switch t {
+	case WildcardDevice, BlockDevice, CharDevice:
+		return true
+	default:
+		return false
+	}
+}
+
+type Rule struct {
+	// Type of device ('c' for char, 'b' for block). If set to 'a', this rule
+	// acts as a wildcard and all fields other than Allow are ignored.
+	Type Type `json:"type"`
+
+	// Major is the device's major number.
+	Major int64 `json:"major"`
+
+	// Minor is the device's minor number.
+	Minor int64 `json:"minor"`
+
+	// Permissions is the set of permissions that this rule applies to (in the
+	// cgroupv1 format -- any combination of "rwm").
+	Permissions Permissions `json:"permissions"`
+
+	// Allow specifies whether this rule is allowed.
+	Allow bool `json:"allow"`
+}
+
+func (d *Rule) CgroupString() string {
+	var (
+		major = strconv.FormatInt(d.Major, 10)
+		minor = strconv.FormatInt(d.Minor, 10)
+	)
+	if d.Major == Wildcard {
+		major = "*"
+	}
+	if d.Minor == Wildcard {
+		minor = "*"
+	}
+	return fmt.Sprintf("%c %s:%s %s", d.Type, major, minor, d.Permissions)
+}
+
+func (d *Rule) Mkdev() (uint64, error) {
+	return mkDev(d)
+}

vendor/github.com/opencontainers/cgroups/devices/config/mknod_unix.go

@@ -0,0 +1,14 @@
+package config
+
+import (
+	"errors"
+
+	"golang.org/x/sys/unix"
+)
+
+func mkDev(d *Rule) (uint64, error) {
+	if d.Major == Wildcard || d.Minor == Wildcard {
+		return 0, errors.New("cannot mkdev() device with wildcards")
+	}
+	return unix.Mkdev(uint32(d.Major), uint32(d.Minor)), nil
+}

vendor/github.com/opencontainers/runc/libcontainer/devices/device_deprecated.go

@@ -0,0 +1,20 @@
+package devices
+
+import "github.com/opencontainers/cgroups/devices/config"
+
+// Deprecated: use [github.com/opencontainers/cgroups/devices/config].
+const (
+	Wildcard       = config.Wildcard
+	WildcardDevice = config.WildcardDevice
+	BlockDevice    = config.BlockDevice
+	CharDevice     = config.CharDevice
+	FifoDevice     = config.FifoDevice
+)
+
+// Deprecated: use [github.com/opencontainers/cgroups/devices/config].
+type (
+	Device      = config.Device
+	Permissions = config.Permissions
+	Type        = config.Type
+	Rule        = config.Rule
+)

vendor/github.com/opencontainers/runc/libcontainer/devices/device_unix.go

@@ -0,0 +1,112 @@
+//go:build !windows
+
+package devices
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+
+	"golang.org/x/sys/unix"
+)
+
+// ErrNotADevice denotes that a file is not a valid linux device.
+var ErrNotADevice = errors.New("not a device node")
+
+// Testing dependencies
+var (
+	unixLstat = unix.Lstat
+	osReadDir = os.ReadDir
+)
+
+// DeviceFromPath takes the path to a device and its cgroup_permissions (which
+// cannot be easily queried) to look up the information about a linux device
+// and returns that information as a Device struct.
+func DeviceFromPath(path, permissions string) (*Device, error) {
+	var stat unix.Stat_t
+	err := unixLstat(path, &stat)
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		devType   Type
+		mode      = stat.Mode
+		devNumber = uint64(stat.Rdev) //nolint:unconvert // Rdev is uint32 on e.g. MIPS.
+		major     = unix.Major(devNumber)
+		minor     = unix.Minor(devNumber)
+	)
+	switch mode & unix.S_IFMT {
+	case unix.S_IFBLK:
+		devType = BlockDevice
+	case unix.S_IFCHR:
+		devType = CharDevice
+	case unix.S_IFIFO:
+		devType = FifoDevice
+	default:
+		return nil, ErrNotADevice
+	}
+	return &Device{
+		Rule: Rule{
+			Type:        devType,
+			Major:       int64(major),
+			Minor:       int64(minor),
+			Permissions: Permissions(permissions),
+		},
+		Path:     path,
+		FileMode: os.FileMode(mode &^ unix.S_IFMT),
+		Uid:      stat.Uid,
+		Gid:      stat.Gid,
+	}, nil
+}
+
+// HostDevices returns all devices that can be found under /dev directory.
+func HostDevices() ([]*Device, error) {
+	return GetDevices("/dev")
+}
+
+// GetDevices recursively traverses a directory specified by path
+// and returns all devices found there.
+func GetDevices(path string) ([]*Device, error) {
+	files, err := osReadDir(path)
+	if err != nil {
+		return nil, err
+	}
+	var out []*Device
+	for _, f := range files {
+		switch {
+		case f.IsDir():
+			switch f.Name() {
+			// ".lxc" & ".lxd-mounts" added to address https://github.com/lxc/lxd/issues/2825
+			// ".udev" added to address https://github.com/opencontainers/runc/issues/2093
+			case "pts", "shm", "fd", "mqueue", ".lxc", ".lxd-mounts", ".udev":
+				continue
+			default:
+				sub, err := GetDevices(filepath.Join(path, f.Name()))
+				if err != nil {
+					return nil, err
+				}
+
+				out = append(out, sub...)
+				continue
+			}
+		case f.Name() == "console":
+			continue
+		}
+		device, err := DeviceFromPath(filepath.Join(path, f.Name()), "rwm")
+		if err != nil {
+			if errors.Is(err, ErrNotADevice) {
+				continue
+			}
+			if os.IsNotExist(err) {
+				continue
+			}
+			return nil, err
+		}
+		if device.Type == FifoDevice {
+			continue
+		}
+		out = append(out, device)
+	}
+	return out, nil
+}

vendor/modules.txt

@@ -37,8 +37,12 @@ github.com/moby/sys/reexec
 # github.com/moby/sys/symlink v0.3.0
 ## explicit; go 1.17
 github.com/moby/sys/symlink
+# github.com/opencontainers/cgroups v0.0.1
+## explicit; go 1.23.0
+github.com/opencontainers/cgroups/devices/config
 # github.com/opencontainers/runc v1.3.0
 ## explicit; go 1.23.0
+github.com/opencontainers/runc/libcontainer/devices
 github.com/opencontainers/runc/libcontainer/exeseal
 github.com/opencontainers/runc/libcontainer/system
 github.com/opencontainers/runc/libcontainer/utils