go-nvlib/pkg/nvmdev/mock.go

/*
 * Copyright (c) 2022, NVIDIA CORPORATION.  All rights reserved.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package nvmdev

import (
	"fmt"
	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci"
	"gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/bytes"
	"os"
	"path/filepath"
)

// MockNvmdev mock mdev device
type MockNvmdev struct {
	*nvmdev
}

var _ Interface = (*MockNvmdev)(nil)

// NewMock creates new mock mediated (vGPU) and parent PCI devices and removes old devices
func NewMock() (mock *MockNvmdev, rerr error) {
	mdevParentsRootDir, err := os.MkdirTemp(os.TempDir(), "")
	if err != nil {
		return nil, err
	}
	defer func() {
		if rerr != nil {
			os.RemoveAll(mdevParentsRootDir)
		}
	}()
	mdevDevicesRootDir, err := os.MkdirTemp(os.TempDir(), "")
	if err != nil {
		return nil, err
	}
	defer func() {
		if rerr != nil {
			os.RemoveAll(mdevDevicesRootDir)
		}
	}()

	mock = &MockNvmdev{
		&nvmdev{mdevParentsRootDir, mdevDevicesRootDir},
	}

	return mock, nil
}

// Cleanup removes the mocked mediated (vGPU) and parent PCI devices root folders
func (m *MockNvmdev) Cleanup() {
	os.RemoveAll(m.mdevParentsRoot)
	os.RemoveAll(m.mdevDevicesRoot)
}

// AddMockA100Parent creates an A100 like parent GPU mock device
func (m *MockNvmdev) AddMockA100Parent(address string, numaNode int) error {
	deviceDir := filepath.Join(m.mdevParentsRoot, address)
	err := os.MkdirAll(deviceDir, 0755)
	if err != nil {
		return err
	}

	vendor, err := os.Create(filepath.Join(deviceDir, "vendor"))
	if err != nil {
		return err
	}
	_, err = vendor.WriteString(fmt.Sprintf("0x%x", nvpci.PCINvidiaVendorID))
	if err != nil {
		return err
	}

	class, err := os.Create(filepath.Join(deviceDir, "class"))
	if err != nil {
		return err
	}
	_, err = class.WriteString(fmt.Sprintf("0x%x", nvpci.PCI3dControllerClass))
	if err != nil {
		return err
	}

	device, err := os.Create(filepath.Join(deviceDir, "device"))
	if err != nil {
		return err
	}
	_, err = device.WriteString("0x20bf")
	if err != nil {
		return err
	}

	_, err = os.Create(filepath.Join(deviceDir, "nvidia"))
	if err != nil {
		return err
	}
	err = os.Symlink(filepath.Join(deviceDir, "nvidia"), filepath.Join(deviceDir, "driver"))
	if err != nil {
		return err
	}

	_, err = os.Create(filepath.Join(deviceDir, "20"))
	if err != nil {
		return err
	}
	err = os.Symlink(filepath.Join(deviceDir, "20"), filepath.Join(deviceDir, "iommu_group"))
	if err != nil {
		return err
	}

	numa, err := os.Create(filepath.Join(deviceDir, "numa_node"))
	if err != nil {
		return err
	}
	_, err = numa.WriteString(fmt.Sprintf("%v", numaNode))
	if err != nil {
		return err
	}

	config, err := os.Create(filepath.Join(deviceDir, "config"))
	if err != nil {
		return err
	}
	_data := make([]byte, nvpci.PCICfgSpaceStandardSize)
	data := bytes.New(&_data)
	data.Write16(0, nvpci.PCINvidiaVendorID)
	data.Write16(2, uint16(0x20bf))
	data.Write8(nvpci.PCIStatusBytePosition, nvpci.PCIStatusCapabilityList)
	_, err = config.Write(*data.Raw())
	if err != nil {
		return err
	}

	bar0 := []uint64{0x00000000c2000000, 0x00000000c2ffffff, 0x0000000000040200}
	resource, err := os.Create(filepath.Join(deviceDir, "resource"))
	if err != nil {
		return err
	}
	_, err = resource.WriteString(fmt.Sprintf("0x%x 0x%x 0x%x", bar0[0], bar0[1], bar0[2]))
	if err != nil {
		return err
	}

	pmcID := uint32(0x170000a1)
	resource0, err := os.Create(filepath.Join(deviceDir, "resource0"))
	if err != nil {
		return err
	}
	_data = make([]byte, bar0[1]-bar0[0]+1)
	data = bytes.New(&_data).LittleEndian()
	data.Write32(0, pmcID)
	_, err = resource0.Write(*data.Raw())
	if err != nil {
		return err
	}

	mdevSupportedTypes := []string{"A100-4C", "A100-5C", "A100-8C", "A100-10C",
		"A100-20C", "A100-40C", "A100-1-5CME", "A100-1-5C", "A100-2-10C", "A100-3-20C",
		"A100-4-20C", "A100-7-40C"}
	mdevSupportedTypesDir := filepath.Join(deviceDir, "mdev_supported_types")
	err = os.MkdirAll(mdevSupportedTypesDir, 0755)
	if err != nil {
		return err
	}
	for i, mdevTypeName := range mdevSupportedTypes {
		mdevTypeDir := filepath.Join(mdevSupportedTypesDir, fmt.Sprintf("nvidia-%d", 500+i))
		err := os.MkdirAll(mdevTypeDir, 0755)
		if err != nil {
			return err
		}
		name, err := os.Create(filepath.Join(mdevTypeDir, "name"))
		if err != nil {
			return err
		}
		_, err = name.WriteString(fmt.Sprintf("NVIDIA %s", mdevTypeName))
		if err != nil {
			return err
		}
		availableInstances, err := os.Create(filepath.Join(mdevTypeDir, "available_instances"))
		if err != nil {
			return err
		}
		_, err = availableInstances.WriteString("1")
		if err != nil {
			return err
		}
	}

	return nil
}

// AddMockA100Mdev creates an A100 like MDEV (vGPU) mock device.
// The corresponding mocked parent A100 device must be created beforehand.
func (m *MockNvmdev) AddMockA100Mdev(uuid string, mdevType string, mdevTypeDir string, parentDeviceDir string) error {
	mdevDeviceDir := filepath.Join(parentDeviceDir, uuid)
	err := os.Mkdir(mdevDeviceDir, 0755)
	if err != nil {
		return err
	}

	parentMdevTypeDir := filepath.Join(parentDeviceDir, "mdev_supported_types", mdevTypeDir)
	err = os.Symlink(parentMdevTypeDir, filepath.Join(mdevDeviceDir, "mdev_type"))
	if err != nil {
		return err
	}

	_, err = os.Create(filepath.Join(mdevDeviceDir, "vfio_mdev"))
	if err != nil {
		return err
	}
	err = os.Symlink(filepath.Join(mdevDeviceDir, "vfio_mdev"), filepath.Join(mdevDeviceDir, "driver"))

	_, err = os.Create(filepath.Join(mdevDeviceDir, "200"))
	if err != nil {
		return err
	}
	err = os.Symlink(filepath.Join(mdevDeviceDir, "200"), filepath.Join(mdevDeviceDir, "iommu_group"))
	if err != nil {
		return err
	}

	err = os.Symlink(mdevDeviceDir, filepath.Join(m.mdevDevicesRoot, uuid))
	if err != nil {
		return err
	}

	return nil
}