diff --git a/pkg/nvpci/nvpci.go b/pkg/nvpci/nvpci.go index ef8cd04..61a8bd3 100644 --- a/pkg/nvpci/nvpci.go +++ b/pkg/nvpci/nvpci.go @@ -48,11 +48,20 @@ type Interface interface { GetGPUs() ([]*NvidiaPCIDevice, error) } +// MemoryResources a more human readable handle +type MemoryResources map[int]*MemoryResource + +// ResourceInterface exposes some higher level functions of resources +type ResourceInterface interface { + GetTotalAddressableMemory(bool) (uint64, uint64) +} + type nvpci struct { pciDevicesRoot string } var _ Interface = (*nvpci)(nil) +var _ ResourceInterface = (*MemoryResources)(nil) // NvidiaPCIDevice represents a PCI device for an NVIDIA product type NvidiaPCIDevice struct { @@ -63,7 +72,7 @@ type NvidiaPCIDevice struct { Device uint16 NumaNode int Config *ConfigSpace - Resources map[int]*MemoryResource + Resources MemoryResources } // IsVGAController if class == 0x300 diff --git a/pkg/nvpci/resources.go b/pkg/nvpci/resources.go index 90b82fc..02a0430 100644 --- a/pkg/nvpci/resources.go +++ b/pkg/nvpci/resources.go @@ -18,6 +18,7 @@ package nvpci import ( "fmt" + "sort" "gitlab.com/nvidia/cloud-native/go-nvlib/pkg/nvpci/mmio" ) @@ -65,3 +66,75 @@ func (mr *MemoryResource) OpenRO() (mmio.Mmio, error) { } return nil, fmt.Errorf("unknown endianness for mmio: %v", err) } + +// From Bit Twiddling Hacks, great resource for all low level bit manipulations +func calcNextPowerOf2(n uint64) uint64 { + n-- + n |= n >> 1 + n |= n >> 2 + n |= n >> 4 + n |= n >> 8 + n |= n >> 16 + n |= n >> 32 + n++ + + return n +} + +// GetTotalAddressableMemory will accumulate the 32bit and 64bit memory windows +// of each BAR and round the value if needed to the next power of 2; first +// return value is the accumulated 32bit addresable memory size the second one +// is the accumulated 64bit addressable memory size in bytes. These values are +// needed to configure virtualized environments. +func (mrs MemoryResources) GetTotalAddressableMemory(roundUp bool) (uint64, uint64) { + const pciIOVNumBAR = 6 + const pciBaseAddressMemTypeMask = 0x06 + const pciBaseAddressMemType32 = 0x00 /* 32 bit address */ + const pciBaseAddressMemType64 = 0x04 /* 64 bit address */ + + // We need to sort the resources so the first 6 entries are the BARs + // How a map is represented in memory is not guaranteed, it is not an + // array. Keys do not have an order. + keys := make([]int, 0, len(mrs)) + for k := range mrs { + keys = append(keys, k) + } + sort.Ints(keys) + + numBAR := 0 + memSize32bit := uint64(0) + memSize64bit := uint64(0) + + for _, key := range keys { + // The PCIe spec only defines 5 BARs per device, we're + // discarding everything after the 5th entry of the resources + // file, see lspci.c + if key >= pciIOVNumBAR || numBAR == pciIOVNumBAR { + break + } + numBAR = numBAR + 1 + + region := mrs[key] + + flags := region.Flags & pciBaseAddressMemTypeMask + memType32bit := flags == pciBaseAddressMemType32 + memType64bit := flags == pciBaseAddressMemType64 + + memSize := (region.End - region.Start) + 1 + + if memType32bit { + memSize32bit = memSize32bit + uint64(memSize) + } + if memType64bit { + memSize64bit = memSize64bit + uint64(memSize) + } + + } + + if roundUp { + memSize32bit = calcNextPowerOf2(memSize32bit) + memSize64bit = calcNextPowerOf2(memSize64bit) + } + + return memSize32bit, memSize64bit +}