Add WSL2 discovery and spec generation

These changes add a wsl discovery mode to the nvidia-ctk cdi generate command.

If wsl mode is enabled, the driver store for the available devices is used as
the source for discovered entities.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

pkg/nvcdi/device-wsl.go

@@ -0,0 +1,37 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/sirupsen/logrus"
+)
+
+const (
+	dxgDeviceNode = "/dev/dxg"
+)
+
+// newDXGDeviceDiscoverer returns a Discoverer for DXG devices under WSL2.
+func newDXGDeviceDiscoverer(logger *logrus.Logger, driverRoot string) discover.Discover {
+	deviceNodes := discover.NewCharDeviceDiscoverer(
+		logger,
+		[]string{dxgDeviceNode},
+		driverRoot,
+	)
+
+	return deviceNodes
+}

pkg/nvcdi/driver-wsl.go

@@ -0,0 +1,106 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+	"path/filepath"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/discover"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/dxcore"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/lookup"
+	"github.com/sirupsen/logrus"
+)
+
+var requiredDriverStoreFiles = []string{
+	"libcuda.so.1.1",                /* Core library for cuda support */
+	"libcuda_loader.so",             /* Core library for cuda support on WSL */
+	"libnvidia-ptxjitcompiler.so.1", /* Core library for PTX Jit support */
+	"libnvidia-ml.so.1",             /* Core library for nvml */
+	"libnvidia-ml_loader.so",        /* Core library for nvml on WSL */
+	"libdxcore.so",                  /* Core library for dxcore support */
+	"nvcubins.bin",                  /* Binary containing GPU code for cuda */
+	"nvidia-smi",                    /* nvidia-smi binary*/
+}
+
+// newWSLDriverDiscoverer returns a Discoverer for WSL2 drivers.
+func newWSLDriverDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string) (discover.Discover, error) {
+	err := dxcore.Init()
+	if err != nil {
+		return nil, fmt.Errorf("failed to initialize dxcore: %v", err)
+	}
+	defer dxcore.Shutdown()
+
+	driverStorePaths := dxcore.GetDriverStorePaths()
+	if len(driverStorePaths) == 0 {
+		return nil, fmt.Errorf("no driver store paths found")
+	}
+	logger.Infof("Using WSL driver store paths: %v", driverStorePaths)
+
+	return newWSLDriverStoreDiscoverer(logger, driverRoot, nvidiaCTKPath, driverStorePaths)
+}
+
+// newWSLDriverStoreDiscoverer returns a Discoverer for WSL2 drivers in the driver store associated with a dxcore adapter.
+func newWSLDriverStoreDiscoverer(logger *logrus.Logger, driverRoot string, nvidiaCTKPath string, driverStorePaths []string) (discover.Discover, error) {
+	var searchPaths []string
+	seen := make(map[string]bool)
+	for _, path := range driverStorePaths {
+		if seen[path] {
+			continue
+		}
+		searchPaths = append(searchPaths, path)
+	}
+	if len(searchPaths) > 1 {
+		logger.Warnf("Found multiple driver store paths: %v", searchPaths)
+	}
+	driverStorePath := searchPaths[0]
+	searchPaths = append(searchPaths, "/usr/lib/wsl/lib")
+
+	libraries := discover.NewMounts(
+		logger,
+		lookup.NewFileLocator(
+			lookup.WithLogger(logger),
+			lookup.WithSearchPaths(
+				searchPaths...,
+			),
+			lookup.WithCount(1),
+		),
+		driverRoot,
+		requiredDriverStoreFiles,
+	)
+
+	// On WSL2 the driver store location is used unchanged.
+	// For this reason we need to create a symlink from /usr/bin/nvidia-smi to the nvidia-smi binary in the driver store.
+	target := filepath.Join(driverStorePath, "nvidia-smi")
+	link := "/usr/bin/nvidia-smi"
+	links := []string{fmt.Sprintf("%s::%s", target, link)}
+	symlinkHook := discover.CreateCreateSymlinkHook(nvidiaCTKPath, links)
+
+	cfg := &discover.Config{
+		DriverRoot:    driverRoot,
+		NvidiaCTKPath: nvidiaCTKPath,
+	}
+	ldcacheHook, _ := discover.NewLDCacheUpdateHook(logger, libraries, cfg)
+
+	d := discover.Merge(
+		libraries,
+		symlinkHook,
+		ldcacheHook,
+	)
+
+	return d, nil
+}

pkg/nvcdi/lib-wsl.go

@@ -0,0 +1,76 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvcdi
+
+import (
+	"fmt"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/edits"
+	"github.com/container-orchestrated-devices/container-device-interface/pkg/cdi"
+	"github.com/container-orchestrated-devices/container-device-interface/specs-go"
+	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvlib/device"
+)
+
+type wsllib nvcdilib
+
+var _ Interface = (*wsllib)(nil)
+
+// GetAllDeviceSpecs returns the device specs for all available devices.
+func (l *wsllib) GetAllDeviceSpecs() ([]specs.Device, error) {
+	device := newDXGDeviceDiscoverer(l.logger, l.driverRoot)
+	deviceEdits, err := edits.FromDiscoverer(device)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container edits for DXG device: %v", err)
+	}
+
+	deviceSpec := specs.Device{
+		Name:           "all",
+		ContainerEdits: *deviceEdits.ContainerEdits,
+	}
+
+	return []specs.Device{deviceSpec}, nil
+}
+
+// GetCommonEdits generates a CDI specification that can be used for ANY devices
+func (l *wsllib) GetCommonEdits() (*cdi.ContainerEdits, error) {
+	driver, err := newWSLDriverDiscoverer(l.logger, l.driverRoot, l.nvidiaCTKPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create discoverer for WSL driver: %v", err)
+	}
+
+	return edits.FromDiscoverer(driver)
+}
+
+// GetGPUDeviceEdits generates a CDI specification that can be used for GPU devices
+func (l *wsllib) GetGPUDeviceEdits(device.Device) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetGPUDeviceEdits is not supported on WSL")
+}
+
+// GetGPUDeviceSpecs returns the CDI device specs for the full GPU represented by 'device'.
+func (l *wsllib) GetGPUDeviceSpecs(i int, d device.Device) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetGPUDeviceSpecs is not supported on WSL")
+}
+
+// GetMIGDeviceEdits generates a CDI specification that can be used for MIG devices
+func (l *wsllib) GetMIGDeviceEdits(device.Device, device.MigDevice) (*cdi.ContainerEdits, error) {
+	return nil, fmt.Errorf("GetMIGDeviceEdits is not supported on WSL")
+}
+
+// GetMIGDeviceSpecs returns the CDI device specs for the full MIG represented by 'device'.
+func (l *wsllib) GetMIGDeviceSpecs(int, device.Device, int, device.MigDevice) (*specs.Device, error) {
+	return nil, fmt.Errorf("GetMIGDeviceSpecs is not supported on WSL")
+}

pkg/nvcdi/lib.go

@@ -64,6 +64,8 @@ func New(opts ...Option) Interface {
 		}
 
 		return (*nvmllib)(l)
+	case "wsl":
+		return (*wsllib)(l)
 	}
 
 	// TODO: We want an error here.