Merge branch 'CNT-4645/add-nvswitch-devices' into 'main'

Add support for NVIDIA_NVSWITCH envvar to inject nvidia-nvswitch device nodes

See merge request nvidia/container-toolkit/container-toolkit!502

CHANGELOG.md

@@ -5,6 +5,7 @@
 * Include `nvidia/nvoptix.bin` in list of graphics mounts.
 * Include `vulkan/icd.d/nvidia_layers.json` in list of graphics mounts.
 * Add support for `--library-search-paths` to `nvidia-ctk cdi generate` command.
+* Add support for injecting /dev/nvidia-nvswitch* devices if the NVIDIA_NVSWITCH=enabled envvar is specified.
 
 * [libnvidia-container] Fix device permission check when using cgroupv2 (fixes #227)
 

internal/discover/nvswitch.go

@@ -0,0 +1,33 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package discover
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+// NewNvSwitchDiscoverer creates a discoverer for NVSWITCH devices.
+func NewNvSwitchDiscoverer(logger logger.Interface, devRoot string) (Discover, error) {
+	devices := NewCharDeviceDiscoverer(
+		logger,
+		devRoot,
+		[]string{
+			"/dev/nvidia-nvswitchctl",
+			"/dev/nvidia-nvswitch*",
+		},
+	)
+
+	return devices, nil
+}

internal/modifier/gated.go

@@ -0,0 +1,79 @@
+/**
+# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package modifier
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
+)
+
+const (
+	nvidiaGDSEnvvar      = "NVIDIA_GDS"
+	nvidiaMOFEDEnvvar    = "NVIDIA_MOFED"
+	nvidiaNVSWITCHEnvvar = "NVIDIA_NVSWITCH"
+)
+
+// NewFeatureGatedModifier creates the modifiers for optional features.
+// These include:
+//
+//	NVIDIA_GDS=enabled
+//	NVIDIA_MOFED=enabled
+//	NVIDIA_NVSWITCH=enabled
+//
+// If not devices are selected, no changes are made.
+func NewFeatureGatedModifier(logger logger.Interface, cfg *config.Config, image image.CUDA) (oci.SpecModifier, error) {
+	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
+		logger.Infof("No modification required; no devices requested")
+		return nil, nil
+	}
+
+	var discoverers []discover.Discover
+
+	driverRoot := cfg.NVIDIAContainerCLIConfig.Root
+	devRoot := cfg.NVIDIAContainerCLIConfig.Root
+
+	if image.Getenv(nvidiaGDSEnvvar) == "enabled" {
+		d, err := discover.NewGDSDiscoverer(logger, driverRoot, devRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to construct discoverer for GDS devices: %w", err)
+		}
+		discoverers = append(discoverers, d)
+	}
+
+	if image.Getenv(nvidiaMOFEDEnvvar) == "enabled" {
+		d, err := discover.NewMOFEDDiscoverer(logger, devRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to construct discoverer for MOFED devices: %w", err)
+		}
+		discoverers = append(discoverers, d)
+	}
+
+	if image.Getenv(nvidiaNVSWITCHEnvvar) == "enabled" {
+		d, err := discover.NewNvSwitchDiscoverer(logger, devRoot)
+		if err != nil {
+			return nil, fmt.Errorf("failed to construct discoverer for NVSWITCH devices: %w", err)
+		}
+		discoverers = append(discoverers, d)
+	}
+
+	return NewModifierFromDiscoverer(logger, discover.Merge(discoverers...))
+}

internal/modifier/gds.go

@@ -1,53 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-)
-
-const (
-	nvidiaGDSEnvvar = "NVIDIA_GDS"
-)
-
-// NewGDSModifier creates the modifiers for GDS devices.
-// If the spec does not contain the NVIDIA_GDS=enabled environment variable no changes are made.
-func NewGDSModifier(logger logger.Interface, cfg *config.Config, image image.CUDA) (oci.SpecModifier, error) {
-	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
-		logger.Infof("No modification required; no devices requested")
-		return nil, nil
-	}
-
-	if image.Getenv(nvidiaGDSEnvvar) != "enabled" {
-		return nil, nil
-	}
-
-	driverRoot := cfg.NVIDIAContainerCLIConfig.Root
-	devRoot := cfg.NVIDIAContainerCLIConfig.Root
-	d, err := discover.NewGDSDiscoverer(logger, driverRoot, devRoot)
-	if err != nil {
-		return nil, fmt.Errorf("failed to construct discoverer for GDS devices: %v", err)
-	}
-
-	return NewModifierFromDiscoverer(logger, d)
-}

internal/modifier/mofed.go

@@ -1,51 +0,0 @@
-/**
-# Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-**/
-
-package modifier
-
-import (
-	"fmt"
-
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/config/image"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
-	"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
-)
-
-const (
-	nvidiaMOFEDEnvvar = "NVIDIA_MOFED"
-)
-
-// NewMOFEDModifier creates the modifiers for MOFED devices.
-// If the spec does not contain the NVIDIA_MOFED=enabled environment variable no changes are made.
-func NewMOFEDModifier(logger logger.Interface, cfg *config.Config, image image.CUDA) (oci.SpecModifier, error) {
-	if devices := image.DevicesFromEnvvars(visibleDevicesEnvvar); len(devices.List()) == 0 {
-		logger.Infof("No modification required; no devices requested")
-		return nil, nil
-	}
-
-	if image.Getenv(nvidiaMOFEDEnvvar) != "enabled" {
-		return nil, nil
-	}
-
-	d, err := discover.NewMOFEDDiscoverer(logger, cfg.NVIDIAContainerCLIConfig.Root)
-	if err != nil {
-		return nil, fmt.Errorf("failed to construct discoverer for MOFED devices: %v", err)
-	}
-
-	return NewModifierFromDiscoverer(logger, d)
-}

internal/runtime/runtime_factory.go

@@ -87,12 +87,7 @@ func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Sp
 		return nil, err
 	}
 
-	gdsModifier, err := modifier.NewGDSModifier(logger, cfg, image)
+	featureModifier, err := modifier.NewFeatureGatedModifier(logger, cfg, image)
-	if err != nil {
-		return nil, err
-	}
-
-	mofedModifier, err := modifier.NewMOFEDModifier(logger, cfg, image)
 	if err != nil {
 		return nil, err
 	}
@@ -100,8 +95,7 @@ func newSpecModifier(logger logger.Interface, cfg *config.Config, ociSpec oci.Sp
 	modifiers := modifier.Merge(
 		modeModifier,
 		graphicsModifier,
-		gdsModifier,
+		featureModifier,
-		mofedModifier,
 	)
 	return modifiers, nil
 }