Split internal system package

This changes splits the functionality in the internal system package
into two packages: one for dealing with devices and one for dealing
with kernel modules. This removes ambiguity around the meaning of
driver / device roots in each case.

In each case, a root can be specified where device nodes are created
or kernel modules loaded.

Signed-off-by: Evan Lezar <elezar@nvidia.com>

internal/system/nvdevices/devices.go

@@ -0,0 +1,154 @@
+/**
+# Copyright (c) NVIDIA CORPORATIOm.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvdevices
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+var errInvalidDeviceNode = errors.New("invalid device node")
+
+// Interface provides a set of utilities for interacting with NVIDIA devices on the system.
+type Interface struct {
+	devices.Devices
+
+	logger logger.Interface
+
+	dryRun bool
+	// devRoot is the root directory where device nodes are expected to exist.
+	devRoot string
+
+	mknoder
+}
+
+// New constructs a new Interface struct with the specified options.
+func New(opts ...Option) (*Interface, error) {
+	i := &Interface{}
+	for _, opt := range opts {
+		opt(i)
+	}
+
+	if i.logger == nil {
+		i.logger = logger.New()
+	}
+	if i.devRoot == "" {
+		i.devRoot = "/"
+	}
+	if i.Devices == nil {
+		devices, err := devices.GetNVIDIADevices()
+		if err != nil {
+			return nil, fmt.Errorf("failed to create devices info: %v", err)
+		}
+		i.Devices = devices
+	}
+
+	if i.dryRun {
+		i.mknoder = &mknodLogger{i.logger}
+	} else {
+		i.mknoder = &mknodUnix{}
+	}
+	return i, nil
+}
+
+// CreateNVIDIAControlDevices creates the NVIDIA control device nodes at the configured devRoot.
+func (m *Interface) CreateNVIDIAControlDevices() error {
+	controlNodes := []string{"nvidiactl", "nvidia-modeset", "nvidia-uvm", "nvidia-uvm-tools"}
+	for _, node := range controlNodes {
+		err := m.CreateNVIDIADevice(node)
+		if err != nil {
+			return fmt.Errorf("failed to create device node %s: %w", node, err)
+		}
+	}
+	return nil
+}
+
+// CreateNVIDIADevice creates the specified NVIDIA device node at the configured devRoot.
+func (m *Interface) CreateNVIDIADevice(node string) error {
+	node = filepath.Base(node)
+	if !strings.HasPrefix(node, "nvidia") {
+		return fmt.Errorf("invalid device node %q: %w", node, errInvalidDeviceNode)
+	}
+
+	major, err := m.Major(node)
+	if err != nil {
+		return fmt.Errorf("failed to determine major: %w", err)
+	}
+
+	minor, err := m.Minor(node)
+	if err != nil {
+		return fmt.Errorf("failed to determine minor: %w", err)
+	}
+
+	return m.createDeviceNode(filepath.Join("dev", node), int(major), int(minor))
+}
+
+// createDeviceNode creates the specified device node with the require major and minor numbers.
+// If a devRoot is configured, this is prepended to the path.
+func (m *Interface) createDeviceNode(path string, major int, minor int) error {
+	path = filepath.Join(m.devRoot, path)
+	if _, err := os.Stat(path); err == nil {
+		m.logger.Infof("Skipping: %s already exists", path)
+		return nil
+	} else if !os.IsNotExist(err) {
+		return fmt.Errorf("failed to stat %s: %v", path, err)
+	}
+
+	return m.Mknode(path, major, minor)
+}
+
+// Major returns the major number for the specified NVIDIA device node.
+// If the device node is not supported, an error is returned.
+func (m *Interface) Major(node string) (int64, error) {
+	var valid bool
+	var major devices.Major
+	switch node {
+	case "nvidia-uvm", "nvidia-uvm-tools":
+		major, valid = m.Get(devices.NVIDIAUVM)
+	case "nvidia-modeset", "nvidiactl":
+		major, valid = m.Get(devices.NVIDIAGPU)
+	}
+
+	if valid {
+		return int64(major), nil
+	}
+
+	return 0, errInvalidDeviceNode
+}
+
+// Minor returns the minor number for the specified NVIDIA device node.
+// If the device node is not supported, an error is returned.
+func (m *Interface) Minor(node string) (int64, error) {
+	switch node {
+	case "nvidia-modeset":
+		return devices.NVIDIAModesetMinor, nil
+	case "nvidia-uvm-tools":
+		return devices.NVIDIAUVMToolsMinor, nil
+	case "nvidia-uvm":
+		return devices.NVIDIAUVMMinor, nil
+	case "nvidiactl":
+		return devices.NVIDIACTLMinor, nil
+	}
+
+	return 0, errInvalidDeviceNode
+}

internal/system/nvdevices/devices_test.go

@@ -0,0 +1,133 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvdevices
+
+import (
+	"errors"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestCreateControlDevices(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	nvidiaDevices := &devices.DevicesMock{
+		GetFunc: func(name devices.Name) (devices.Major, bool) {
+			devices := map[devices.Name]devices.Major{
+				"nvidia-frontend": 195,
+				"nvidia-uvm":      243,
+			}
+			return devices[name], true
+		},
+	}
+
+	mknodeError := errors.New("mknode error")
+
+	testCases := []struct {
+		description   string
+		root          string
+		devices       devices.Devices
+		mknodeError   error
+		expectedError error
+		expectedCalls []struct {
+			S  string
+			N1 int
+			N2 int
+		}
+	}{
+		{
+			description: "no root specified",
+			root:        "",
+			devices:     nvidiaDevices,
+			mknodeError: nil,
+			expectedCalls: []struct {
+				S  string
+				N1 int
+				N2 int
+			}{
+				{"/dev/nvidiactl", 195, 255},
+				{"/dev/nvidia-modeset", 195, 254},
+				{"/dev/nvidia-uvm", 243, 0},
+				{"/dev/nvidia-uvm-tools", 243, 1},
+			},
+		},
+		{
+			description: "some root specified",
+			root:        "/some/root",
+			devices:     nvidiaDevices,
+			mknodeError: nil,
+			expectedCalls: []struct {
+				S  string
+				N1 int
+				N2 int
+			}{
+				{"/some/root/dev/nvidiactl", 195, 255},
+				{"/some/root/dev/nvidia-modeset", 195, 254},
+				{"/some/root/dev/nvidia-uvm", 243, 0},
+				{"/some/root/dev/nvidia-uvm-tools", 243, 1},
+			},
+		},
+		{
+			description:   "mknod error returns error",
+			devices:       nvidiaDevices,
+			mknodeError:   mknodeError,
+			expectedError: mknodeError,
+			// We expect the first call to this to fail, and the rest to be skipped
+			expectedCalls: []struct {
+				S  string
+				N1 int
+				N2 int
+			}{
+				{"/dev/nvidiactl", 195, 255},
+			},
+		},
+		{
+			description: "missing major returns error",
+			devices: &devices.DevicesMock{
+				GetFunc: func(name devices.Name) (devices.Major, bool) {
+					return 0, false
+				},
+			},
+			expectedError: errInvalidDeviceNode,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			mknode := &mknoderMock{
+				MknodeFunc: func(string, int, int) error {
+					return tc.mknodeError
+				},
+			}
+
+			d, _ := New(
+				WithLogger(logger),
+				WithDevRoot(tc.root),
+				WithDevices(tc.devices),
+			)
+			d.mknoder = mknode
+
+			err := d.CreateNVIDIAControlDevices()
+			require.ErrorIs(t, err, tc.expectedError)
+			require.EqualValues(t, tc.expectedCalls, mknode.MknodeCalls())
+		})
+	}
+
+}

internal/system/nvdevices/mknod.go

@@ -0,0 +1,46 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvdevices
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+	"golang.org/x/sys/unix"
+)
+
+//go:generate moq -stub -out mknod_mock.go . mknoder
+type mknoder interface {
+	Mknode(string, int, int) error
+}
+
+type mknodLogger struct {
+	logger.Interface
+}
+
+func (m *mknodLogger) Mknode(path string, major, minor int) error {
+	m.Infof("Running: mknod --mode=0666 %s c %d %d", path, major, minor)
+	return nil
+}
+
+type mknodUnix struct{}
+
+func (m *mknodUnix) Mknode(path string, major, minor int) error {
+	err := unix.Mknod(path, unix.S_IFCHR, int(unix.Mkdev(uint32(major), uint32(minor))))
+	if err != nil {
+		return err
+	}
+	return unix.Chmod(path, 0666)
+}

internal/system/nvdevices/mknod_mock.go

@@ -0,0 +1,89 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package nvdevices
+
+import (
+	"sync"
+)
+
+// Ensure, that mknoderMock does implement mknoder.
+// If this is not the case, regenerate this file with moq.
+var _ mknoder = &mknoderMock{}
+
+// mknoderMock is a mock implementation of mknoder.
+//
+//	func TestSomethingThatUsesmknoder(t *testing.T) {
+//
+//		// make and configure a mocked mknoder
+//		mockedmknoder := &mknoderMock{
+//			MknodeFunc: func(s string, n1 int, n2 int) error {
+//				panic("mock out the Mknode method")
+//			},
+//		}
+//
+//		// use mockedmknoder in code that requires mknoder
+//		// and then make assertions.
+//
+//	}
+type mknoderMock struct {
+	// MknodeFunc mocks the Mknode method.
+	MknodeFunc func(s string, n1 int, n2 int) error
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// Mknode holds details about calls to the Mknode method.
+		Mknode []struct {
+			// S is the s argument value.
+			S string
+			// N1 is the n1 argument value.
+			N1 int
+			// N2 is the n2 argument value.
+			N2 int
+		}
+	}
+	lockMknode sync.RWMutex
+}
+
+// Mknode calls MknodeFunc.
+func (mock *mknoderMock) Mknode(s string, n1 int, n2 int) error {
+	callInfo := struct {
+		S  string
+		N1 int
+		N2 int
+	}{
+		S:  s,
+		N1: n1,
+		N2: n2,
+	}
+	mock.lockMknode.Lock()
+	mock.calls.Mknode = append(mock.calls.Mknode, callInfo)
+	mock.lockMknode.Unlock()
+	if mock.MknodeFunc == nil {
+		var (
+			errOut error
+		)
+		return errOut
+	}
+	return mock.MknodeFunc(s, n1, n2)
+}
+
+// MknodeCalls gets all the calls that were made to Mknode.
+// Check the length with:
+//
+//	len(mockedmknoder.MknodeCalls())
+func (mock *mknoderMock) MknodeCalls() []struct {
+	S  string
+	N1 int
+	N2 int
+} {
+	var calls []struct {
+		S  string
+		N1 int
+		N2 int
+	}
+	mock.lockMknode.RLock()
+	calls = mock.calls.Mknode
+	mock.lockMknode.RUnlock()
+	return calls
+}

internal/system/nvdevices/options.go

@@ -0,0 +1,53 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package nvdevices
+
+import (
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/info/proc/devices"
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+)
+
+// Option is a function that sets an option on the Interface struct.
+type Option func(*Interface)
+
+// WithDryRun sets the dry run option for the Interface struct.
+func WithDryRun(dryRun bool) Option {
+	return func(i *Interface) {
+		i.dryRun = dryRun
+	}
+}
+
+// WithLogger sets the logger for the Interface struct.
+func WithLogger(logger logger.Interface) Option {
+	return func(i *Interface) {
+		i.logger = logger
+	}
+}
+
+// WithDevRoot sets the root directory for the NVIDIA device nodes.
+func WithDevRoot(devRoot string) Option {
+	return func(i *Interface) {
+		i.devRoot = devRoot
+	}
+}
+
+// WithDevices sets the devices for the Interface struct.
+func WithDevices(devices devices.Devices) Option {
+	return func(i *Interface) {
+		i.Devices = devices
+	}
+}