UniqAI/nvidia-container-toolkit
1
0
Fork 0
mirror of https://github.com/NVIDIA/nvidia-container-toolkit synced 2025-06-26 18:18:24 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Refactor accepting device lists from volume mounts as a boolean

Browse Source 
Also hard code the "root" path where these volume mounts will be looked
for rather than making it configurable.

Signed-off-by: Kevin Klues <kklues@nvidia.com>
This commit is contained in:
Kevin Klues 2020-08-07 14:24:32 +00:00
parent 928905ce94
commit 7c00385797
8 changed files with 41 additions and 40 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/config.toml.amzn
View File

@ -1,7 +1,7 @@
disable-require = false disable-require = false
#swarm-resource = "DOCKER_RESOURCE_GPU" #swarm-resource = "DOCKER_RESOURCE_GPU"
#accept-nvidia-visible-devices-envvar-when-unprivileged = true #accept-nvidia-visible-devices-envvar-when-unprivileged = true
#look-for-nvidia-visible-devices-as-volume-mounts-under = "/var/run/nvidia-container-devices" #accept-nvidia-visible-devices-as-volume-mounts = false
[nvidia-container-cli] [nvidia-container-cli]
#root = "/run/nvidia/driver" #root = "/run/nvidia/driver"

2
config/config.toml.centos
View File

@ -1,7 +1,7 @@
disable-require = false disable-require = false
#swarm-resource = "DOCKER_RESOURCE_GPU" #swarm-resource = "DOCKER_RESOURCE_GPU"
#accept-nvidia-visible-devices-envvar-when-unprivileged = true #accept-nvidia-visible-devices-envvar-when-unprivileged = true
#look-for-nvidia-visible-devices-as-volume-mounts-under = "/var/run/nvidia-container-devices" #accept-nvidia-visible-devices-as-volume-mounts = false
[nvidia-container-cli] [nvidia-container-cli]
#root = "/run/nvidia/driver" #root = "/run/nvidia/driver"

2
config/config.toml.debian
View File

@ -1,7 +1,7 @@
disable-require = false disable-require = false
#swarm-resource = "DOCKER_RESOURCE_GPU" #swarm-resource = "DOCKER_RESOURCE_GPU"
#accept-nvidia-visible-devices-envvar-when-unprivileged = true #accept-nvidia-visible-devices-envvar-when-unprivileged = true
#look-for-nvidia-visible-devices-as-volume-mounts-under = "/var/run/nvidia-container-devices" #accept-nvidia-visible-devices-as-volume-mounts = false
[nvidia-container-cli] [nvidia-container-cli]
#root = "/run/nvidia/driver" #root = "/run/nvidia/driver"

2
config/config.toml.opensuse-leap
View File

@ -1,7 +1,7 @@
disable-require = false disable-require = false
#swarm-resource = "DOCKER_RESOURCE_GPU" #swarm-resource = "DOCKER_RESOURCE_GPU"
#accept-nvidia-visible-devices-envvar-when-unprivileged = true #accept-nvidia-visible-devices-envvar-when-unprivileged = true
#look-for-nvidia-visible-devices-as-volume-mounts-under = "/var/run/nvidia-container-devices" #accept-nvidia-visible-devices-as-volume-mounts = false
[nvidia-container-cli] [nvidia-container-cli]
#root = "/run/nvidia/driver" #root = "/run/nvidia/driver"

2
config/config.toml.ubuntu
View File

@ -1,7 +1,7 @@
disable-require = false disable-require = false
#swarm-resource = "DOCKER_RESOURCE_GPU" #swarm-resource = "DOCKER_RESOURCE_GPU"
#accept-nvidia-visible-devices-envvar-when-unprivileged = true #accept-nvidia-visible-devices-envvar-when-unprivileged = true
#look-for-nvidia-visible-devices-as-volume-mounts-under = "/var/run/nvidia-container-devices" #accept-nvidia-visible-devices-as-volume-mounts = false
[nvidia-container-cli] [nvidia-container-cli]
#root = "/run/nvidia/driver" #root = "/run/nvidia/driver"

20
pkg/container_config.go
View File

@ -35,6 +35,10 @@ const (
capSysAdmin = "CAP_SYS_ADMIN" capSysAdmin = "CAP_SYS_ADMIN"
) )
const (
deviceListAsVolumeMountsRoot = "/var/run/nvidia-container-devices"
)
type nvidiaConfig struct { type nvidiaConfig struct {
Devices string Devices string
MigConfigDevices string MigConfigDevices string
@ -236,10 +240,10 @@ func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string {
return devices return devices
} }
func getDevicesFromMounts(root string, mounts []Mount) *string { func getDevicesFromMounts(mounts []Mount) *string {
var devices []string var devices []string
for _, m := range mounts { for _, m := range mounts {
root := filepath.Clean(root) root := filepath.Clean(deviceListAsVolumeMountsRoot)
source := filepath.Clean(m.Source) source := filepath.Clean(m.Source)
destination := filepath.Clean(m.Destination) destination := filepath.Clean(m.Destination)
@ -274,14 +278,16 @@ func getDevicesFromMounts(root string, mounts []Mount) *string {
} }
func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string { func getDevices(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool, legacyImage bool) *string {
// Try and get the device list from mount volumes first // If enabled, try and get the device list from volume mounts first
devices := getDevicesFromMounts(*hookConfig.DeviceListVolumeMount, mounts) if hookConfig.AcceptDeviceListAsVolumeMounts {
if devices != nil { devices := getDevicesFromMounts(mounts)
return devices if devices != nil {
return devices
}
} }
// Fallback to reading from the environment variable if privileges are correct // Fallback to reading from the environment variable if privileges are correct
devices = getDevicesFromEnvvar(env, legacyImage) devices := getDevicesFromEnvvar(env, legacyImage)
if devices == nil { if devices == nil {
return nil return nil
} }

31
pkg/container_test.go
View File

@ -454,30 +454,26 @@ func TestGetNvidiaConfig(t *testing.T) {
func TestGetDevicesFromMounts(t *testing.T) { func TestGetDevicesFromMounts(t *testing.T) {
var tests = []struct { var tests = []struct {
description string description string
root string
mounts []Mount mounts []Mount
expectedDevices *string expectedDevices *string
}{ }{
{ {
description: "No mounts", description: "No mounts",
root: defaultDeviceListVolumeMount,
mounts: nil, mounts: nil,
expectedDevices: nil, expectedDevices: nil,
}, },
{ {
description: "Host path is not /dev/null", description: "Host path is not /dev/null",
root: defaultDeviceListVolumeMount,
mounts: []Mount{ mounts: []Mount{
{ {
Source: "/not/dev/null", Source: "/not/dev/null",
Destination: filepath.Join(defaultDeviceListVolumeMount, "GPU0"), Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
}, },
}, },
expectedDevices: nil, expectedDevices: nil,
}, },
{ {
description: "Container path is not prefixed by 'root'", description: "Container path is not prefixed by 'root'",
root: defaultDeviceListVolumeMount,
mounts: []Mount{ mounts: []Mount{
{ {
Source: "/dev/null", Source: "/dev/null",
@ -488,41 +484,38 @@ func TestGetDevicesFromMounts(t *testing.T) {
}, },
{ {
description: "Container path is only 'root'", description: "Container path is only 'root'",
root: defaultDeviceListVolumeMount,
mounts: []Mount{ mounts: []Mount{
{ {
Source: "/dev/null", Source: "/dev/null",
Destination: defaultDeviceListVolumeMount, Destination: deviceListAsVolumeMountsRoot,
}, },
}, },
expectedDevices: nil, expectedDevices: nil,
}, },
{ {
description: "Discover 2 devices", description: "Discover 2 devices",
root: defaultDeviceListVolumeMount,
mounts: []Mount{ mounts: []Mount{
{ {
Source: "/dev/null", Source: "/dev/null",
Destination: filepath.Join(defaultDeviceListVolumeMount, "GPU0"), Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
}, },
{ {
Source: "/dev/null", Source: "/dev/null",
Destination: filepath.Join(defaultDeviceListVolumeMount, "GPU1"), Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
}, },
}, },
expectedDevices: &[]string{"GPU0,GPU1"}[0], expectedDevices: &[]string{"GPU0,GPU1"}[0],
}, },
{ {
description: "Discover 2 devices with slashes in the name", description: "Discover 2 devices with slashes in the name",
root: defaultDeviceListVolumeMount,
mounts: []Mount{ mounts: []Mount{
{ {
Source: "/dev/null", Source: "/dev/null",
Destination: filepath.Join(defaultDeviceListVolumeMount, "GPU0-MIG0/0/1"), Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0-MIG0/0/1"),
}, },
{ {
Source: "/dev/null", Source: "/dev/null",
Destination: filepath.Join(defaultDeviceListVolumeMount, "GPU1-MIG0/0/1"), Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1-MIG0/0/1"),
}, },
}, },
expectedDevices: &[]string{"GPU0-MIG0/0/1,GPU1-MIG0/0/1"}[0], expectedDevices: &[]string{"GPU0-MIG0/0/1,GPU1-MIG0/0/1"}[0],
@ -530,7 +523,7 @@ func TestGetDevicesFromMounts(t *testing.T) {
} }
for _, tc := range tests { for _, tc := range tests {
t.Run(tc.description, func(t *testing.T) { t.Run(tc.description, func(t *testing.T) {
devices := getDevicesFromMounts(tc.root, tc.mounts) devices := getDevicesFromMounts(tc.mounts)
if !reflect.DeepEqual(devices, tc.expectedDevices) { if !reflect.DeepEqual(devices, tc.expectedDevices) {
t.Errorf("Unexpected devices (got: %v, wanted: %v)", *devices, *tc.expectedDevices) t.Errorf("Unexpected devices (got: %v, wanted: %v)", *devices, *tc.expectedDevices)
} }
@ -545,6 +538,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
envvarDevices string envvarDevices string
privileged bool privileged bool
acceptUnprivileged bool acceptUnprivileged bool
acceptMounts bool
expectedDevices *string expectedDevices *string
expectedPanic bool expectedPanic bool
}{ }{
@ -553,16 +547,17 @@ func TestDeviceListSourcePriority(t *testing.T) {
mountDevices: []Mount{ mountDevices: []Mount{
{ {
Source: "/dev/null", Source: "/dev/null",
Destination: filepath.Join(defaultDeviceListVolumeMount, "GPU0"), Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU0"),
}, },
{ {
Source: "/dev/null", Source: "/dev/null",
Destination: filepath.Join(defaultDeviceListVolumeMount, "GPU1"), Destination: filepath.Join(deviceListAsVolumeMountsRoot, "GPU1"),
}, },
}, },
envvarDevices: "GPU2,GPU3", envvarDevices: "GPU2,GPU3",
privileged: false, privileged: false,
acceptUnprivileged: false, acceptUnprivileged: false,
acceptMounts: true,
expectedDevices: &[]string{"GPU0,GPU1"}[0], expectedDevices: &[]string{"GPU0,GPU1"}[0],
}, },
{ {
@ -571,6 +566,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
envvarDevices: "GPU0,GPU1", envvarDevices: "GPU0,GPU1",
privileged: false, privileged: false,
acceptUnprivileged: false, acceptUnprivileged: false,
acceptMounts: true,
expectedPanic: true, expectedPanic: true,
}, },
{ {
@ -579,6 +575,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
envvarDevices: "GPU0,GPU1", envvarDevices: "GPU0,GPU1",
privileged: true, privileged: true,
acceptUnprivileged: false, acceptUnprivileged: false,
acceptMounts: true,
expectedDevices: &[]string{"GPU0,GPU1"}[0], expectedDevices: &[]string{"GPU0,GPU1"}[0],
}, },
{ {
@ -587,6 +584,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
envvarDevices: "GPU0,GPU1", envvarDevices: "GPU0,GPU1",
privileged: false, privileged: false,
acceptUnprivileged: true, acceptUnprivileged: true,
acceptMounts: true,
expectedDevices: &[]string{"GPU0,GPU1"}[0], expectedDevices: &[]string{"GPU0,GPU1"}[0],
}, },
} }
@ -600,6 +598,7 @@ func TestDeviceListSourcePriority(t *testing.T) {
} }
hookConfig := getDefaultHookConfig() hookConfig := getDefaultHookConfig()
hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged hookConfig.AcceptEnvvarUnprivileged = tc.acceptUnprivileged
hookConfig.AcceptDeviceListAsVolumeMounts = tc.acceptMounts
devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false) devices = getDevices(&hookConfig, env, tc.mountDevices, tc.privileged, false)
} }

20
pkg/hook_config.go
View File

@ -13,10 +13,6 @@ const (
driverPath = "/run/nvidia/driver" driverPath = "/run/nvidia/driver"
) )
const (
defaultDeviceListVolumeMount = "/var/run/nvidia-container-devices"
)
var defaultPaths = [...]string{ var defaultPaths = [...]string{
path.Join(driverPath, configPath), path.Join(driverPath, configPath),
configPath, configPath,
@ -38,20 +34,20 @@ type CLIConfig struct {
// HookConfig : options for the nvidia-container-toolkit. // HookConfig : options for the nvidia-container-toolkit.
type HookConfig struct { type HookConfig struct {
DisableRequire bool `toml:"disable-require"` DisableRequire bool `toml:"disable-require"`
SwarmResource *string `toml:"swarm-resource"` SwarmResource *string `toml:"swarm-resource"`
AcceptEnvvarUnprivileged bool `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"` AcceptEnvvarUnprivileged bool `toml:"accept-nvidia-visible-devices-envvar-when-unprivileged"`
DeviceListVolumeMount *string `toml:"look-for-nvidia-visible-devices-as-volume-mounts-under"` AcceptDeviceListAsVolumeMounts bool `toml:"accept-nvidia-visible-devices-as-volume-mounts"`
NvidiaContainerCLI CLIConfig `toml:"nvidia-container-cli"` NvidiaContainerCLI CLIConfig `toml:"nvidia-container-cli"`
} }
func getDefaultHookConfig() (config HookConfig) { func getDefaultHookConfig() (config HookConfig) {
return HookConfig{ return HookConfig{
DisableRequire: false, DisableRequire: false,
SwarmResource: nil, SwarmResource: nil,
AcceptEnvvarUnprivileged: true, AcceptEnvvarUnprivileged: true,
DeviceListVolumeMount: &[]string{defaultDeviceListVolumeMount}[0], AcceptDeviceListAsVolumeMounts: false,
NvidiaContainerCLI: CLIConfig{ NvidiaContainerCLI: CLIConfig{
Root: nil, Root: nil,
Path: nil, Path: nil,