From 35982e51bf1f81cbf9de3479fdc2a2410e413fb5 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 25 Jul 2022 10:01:33 +0200 Subject: [PATCH 1/7] Move toolkit options to struct Signed-off-by: Evan Lezar --- tools/container/toolkit/toolkit.go | 44 ++++++++++++++++++------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/tools/container/toolkit/toolkit.go b/tools/container/toolkit/toolkit.go index 4f215942..4b477e69 100644 --- a/tools/container/toolkit/toolkit.go +++ b/tools/container/toolkit/toolkit.go @@ -40,12 +40,18 @@ const ( ) var toolkitDirArg string -var nvidiaDriverRootFlag string -var nvidiaContainerRuntimeDebugFlag string -var nvidiaContainerRuntimeLogLevelFlag string -var nvidiaContainerCLIDebugFlag string + +type options struct { + DriverRoot string + ContainerRuntimeDebug string + ContainerRuntimeLogLevel string + ContainerCLIDebug string +} func main() { + + opts := options{} + // Create the top-level CLI c := cli.NewApp() c.Name = "toolkit" @@ -58,7 +64,9 @@ func main() { install.Usage = "Install the components of the NVIDIA container toolkit" install.ArgsUsage = "" install.Before = parseArgs - install.Action = Install + install.Action = func(c *cli.Context) error { + return Install(c, &opts) + } // Create the 'delete' command delete := cli.Command{} @@ -78,24 +86,24 @@ func main() { &cli.StringFlag{ Name: "nvidia-driver-root", Value: DefaultNvidiaDriverRoot, - Destination: &nvidiaDriverRootFlag, + Destination: &opts.DriverRoot, EnvVars: []string{"NVIDIA_DRIVER_ROOT"}, }, &cli.StringFlag{ Name: "nvidia-container-runtime-debug", Usage: "Specify the location of the debug log file for the NVIDIA Container Runtime", - Destination: &nvidiaContainerRuntimeDebugFlag, + Destination: &opts.ContainerRuntimeDebug, EnvVars: []string{"NVIDIA_CONTAINER_RUNTIME_DEBUG"}, }, &cli.StringFlag{ Name: "nvidia-container-runtime-debug-log-level", - Destination: &nvidiaContainerRuntimeLogLevelFlag, + Destination: &opts.ContainerRuntimeLogLevel, EnvVars: []string{"NVIDIA_CONTAINER_RUNTIME_LOG_LEVEL"}, }, &cli.StringFlag{ Name: "nvidia-container-cli-debug", Usage: "Specify the location of the debug log file for the NVIDIA Container CLI", - Destination: &nvidiaContainerCLIDebugFlag, + Destination: &opts.ContainerCLIDebug, EnvVars: []string{"NVIDIA_CONTAINER_CLI_DEBUG"}, }, } @@ -135,7 +143,7 @@ func Delete(cli *cli.Context) error { // Install installs the components of the NVIDIA container toolkit. // Any existing installation is removed. -func Install(cli *cli.Context) error { +func Install(cli *cli.Context, opts *options) error { log.Infof("Installing NVIDIA container toolkit to '%v'", toolkitDirArg) log.Infof("Removing existing NVIDIA container toolkit installation") @@ -157,7 +165,7 @@ func Install(cli *cli.Context) error { return fmt.Errorf("error installing NVIDIA container library: %v", err) } - err = installContainerRuntimes(toolkitDirArg, nvidiaDriverRootFlag) + err = installContainerRuntimes(toolkitDirArg, opts.DriverRoot) if err != nil { return fmt.Errorf("error installing NVIDIA container runtime: %v", err) } @@ -172,7 +180,7 @@ func Install(cli *cli.Context) error { return fmt.Errorf("error installing NVIDIA container runtime hook: %v", err) } - err = installToolkitConfig(toolkitConfigPath, nvidiaDriverRootFlag, nvidiaContainerCliExecutable) + err = installToolkitConfig(toolkitConfigPath, nvidiaContainerCliExecutable, opts) if err != nil { return fmt.Errorf("error installing NVIDIA container toolkit config: %v", err) } @@ -230,7 +238,7 @@ func installLibrary(libName string, toolkitDir string) error { // installToolkitConfig installs the config file for the NVIDIA container toolkit ensuring // that the settings are updated to match the desired install and nvidia driver directories. -func installToolkitConfig(toolkitConfigPath string, nvidiaDriverDir string, nvidiaContainerCliExecutablePath string) error { +func installToolkitConfig(toolkitConfigPath string, nvidiaContainerCliExecutablePath string, opts *options) error { log.Infof("Installing NVIDIA container toolkit config '%v'", toolkitConfigPath) config, err := toml.LoadFile(nvidiaContainerToolkitConfigSource) @@ -253,17 +261,17 @@ func installToolkitConfig(toolkitConfigPath string, nvidiaDriverDir string, nvid ldconfigPath := fmt.Sprintf("%s", config.GetPath(nvidiaContainerCliKey("ldconfig"))) // Use the driver run root as the root: - driverLdconfigPath := "@" + filepath.Join(nvidiaDriverDir, strings.TrimPrefix(ldconfigPath, "@/")) + driverLdconfigPath := "@" + filepath.Join(opts.DriverRoot, strings.TrimPrefix(ldconfigPath, "@/")) - config.SetPath(nvidiaContainerCliKey("root"), nvidiaDriverDir) + config.SetPath(nvidiaContainerCliKey("root"), opts.DriverRoot) config.SetPath(nvidiaContainerCliKey("path"), nvidiaContainerCliExecutablePath) config.SetPath(nvidiaContainerCliKey("ldconfig"), driverLdconfigPath) // Set the debug options if selected debugOptions := map[string]string{ - "nvidia-container-runtime.debug": nvidiaContainerRuntimeDebugFlag, - "nvidia-container-runtime.log-level": nvidiaContainerRuntimeLogLevelFlag, - "nvidia-container-cli.debug": nvidiaContainerCLIDebugFlag, + "nvidia-container-runtime.debug": opts.ContainerRuntimeDebug, + "nvidia-container-runtime.log-level": opts.ContainerRuntimeLogLevel, + "nvidia-container-cli.debug": opts.ContainerCLIDebug, } for key, value := range debugOptions { if value == "" { From ab3f05cf629c53c695a9562676fec5ef8d537a50 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 25 Jul 2022 10:08:36 +0200 Subject: [PATCH 2/7] Move global toolkitDir to options struct Signed-off-by: Evan Lezar --- tools/container/toolkit/toolkit.go | 41 +++++++++++++++++------------- 1 file changed, 23 insertions(+), 18 deletions(-) diff --git a/tools/container/toolkit/toolkit.go b/tools/container/toolkit/toolkit.go index 4b477e69..27ab152c 100644 --- a/tools/container/toolkit/toolkit.go +++ b/tools/container/toolkit/toolkit.go @@ -39,13 +39,12 @@ const ( configFilename = "config.toml" ) -var toolkitDirArg string - type options struct { DriverRoot string ContainerRuntimeDebug string ContainerRuntimeLogLevel string ContainerCLIDebug string + toolkitDir string } func main() { @@ -63,7 +62,9 @@ func main() { install.Name = "install" install.Usage = "Install the components of the NVIDIA container toolkit" install.ArgsUsage = "" - install.Before = parseArgs + install.Before = func(c *cli.Context) error { + return parseArgs(c, &opts) + } install.Action = func(c *cli.Context) error { return Install(c, &opts) } @@ -73,8 +74,12 @@ func main() { delete.Name = "delete" delete.Usage = "Delete the NVIDIA container toolkit" delete.ArgsUsage = "" - delete.Before = parseArgs - delete.Action = Delete + delete.Before = func(c *cli.Context) error { + return parseArgs(c, &opts) + } + delete.Action = func(c *cli.Context) error { + return Delete(c, &opts) + } // Register the subcommand with the top-level CLI c.Commands = []*cli.Command{ @@ -118,23 +123,23 @@ func main() { } // parseArgs parses the command line arguments to the CLI -func parseArgs(c *cli.Context) error { +func parseArgs(c *cli.Context, opts *options) error { args := c.Args() log.Infof("Parsing arguments: %v", args.Slice()) if c.NArg() != 1 { return fmt.Errorf("incorrect number of arguments") } - toolkitDirArg = args.Get(0) + opts.toolkitDir = args.Get(0) log.Infof("Successfully parsed arguments") return nil } // Delete removes the NVIDIA container toolkit -func Delete(cli *cli.Context) error { - log.Infof("Deleting NVIDIA container toolkit from '%v'", toolkitDirArg) - err := os.RemoveAll(toolkitDirArg) +func Delete(cli *cli.Context, opts *options) error { + log.Infof("Deleting NVIDIA container toolkit from '%v'", opts.toolkitDir) + err := os.RemoveAll(opts.toolkitDir) if err != nil { return fmt.Errorf("error deleting toolkit directory: %v", err) } @@ -144,38 +149,38 @@ func Delete(cli *cli.Context) error { // Install installs the components of the NVIDIA container toolkit. // Any existing installation is removed. func Install(cli *cli.Context, opts *options) error { - log.Infof("Installing NVIDIA container toolkit to '%v'", toolkitDirArg) + log.Infof("Installing NVIDIA container toolkit to '%v'", opts.toolkitDir) log.Infof("Removing existing NVIDIA container toolkit installation") - err := os.RemoveAll(toolkitDirArg) + err := os.RemoveAll(opts.toolkitDir) if err != nil { return fmt.Errorf("error removing toolkit directory: %v", err) } - toolkitConfigDir := filepath.Join(toolkitDirArg, ".config", "nvidia-container-runtime") + toolkitConfigDir := filepath.Join(opts.toolkitDir, ".config", "nvidia-container-runtime") toolkitConfigPath := filepath.Join(toolkitConfigDir, configFilename) - err = createDirectories(toolkitDirArg, toolkitConfigDir) + err = createDirectories(opts.toolkitDir, toolkitConfigDir) if err != nil { return fmt.Errorf("could not create required directories: %v", err) } - err = installContainerLibraries(toolkitDirArg) + err = installContainerLibraries(opts.toolkitDir) if err != nil { return fmt.Errorf("error installing NVIDIA container library: %v", err) } - err = installContainerRuntimes(toolkitDirArg, opts.DriverRoot) + err = installContainerRuntimes(opts.toolkitDir, opts.DriverRoot) if err != nil { return fmt.Errorf("error installing NVIDIA container runtime: %v", err) } - nvidiaContainerCliExecutable, err := installContainerCLI(toolkitDirArg) + nvidiaContainerCliExecutable, err := installContainerCLI(opts.toolkitDir) if err != nil { return fmt.Errorf("error installing NVIDIA container CLI: %v", err) } - _, err = installRuntimeHook(toolkitDirArg, toolkitConfigPath) + _, err = installRuntimeHook(opts.toolkitDir, toolkitConfigPath) if err != nil { return fmt.Errorf("error installing NVIDIA container runtime hook: %v", err) } From feedc912e474332d8551d47ec188a1a867917fb3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 25 Jul 2022 10:12:47 +0200 Subject: [PATCH 3/7] Rename toolkitDir toolkitRoot Signed-off-by: Evan Lezar --- tools/container/toolkit/toolkit.go | 54 +++++++++++++++--------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/tools/container/toolkit/toolkit.go b/tools/container/toolkit/toolkit.go index 27ab152c..23d892f3 100644 --- a/tools/container/toolkit/toolkit.go +++ b/tools/container/toolkit/toolkit.go @@ -44,7 +44,7 @@ type options struct { ContainerRuntimeDebug string ContainerRuntimeLogLevel string ContainerCLIDebug string - toolkitDir string + toolkitRoot string } func main() { @@ -130,7 +130,7 @@ func parseArgs(c *cli.Context, opts *options) error { if c.NArg() != 1 { return fmt.Errorf("incorrect number of arguments") } - opts.toolkitDir = args.Get(0) + opts.toolkitRoot = args.Get(0) log.Infof("Successfully parsed arguments") return nil @@ -138,8 +138,8 @@ func parseArgs(c *cli.Context, opts *options) error { // Delete removes the NVIDIA container toolkit func Delete(cli *cli.Context, opts *options) error { - log.Infof("Deleting NVIDIA container toolkit from '%v'", opts.toolkitDir) - err := os.RemoveAll(opts.toolkitDir) + log.Infof("Deleting NVIDIA container toolkit from '%v'", opts.toolkitRoot) + err := os.RemoveAll(opts.toolkitRoot) if err != nil { return fmt.Errorf("error deleting toolkit directory: %v", err) } @@ -149,38 +149,38 @@ func Delete(cli *cli.Context, opts *options) error { // Install installs the components of the NVIDIA container toolkit. // Any existing installation is removed. func Install(cli *cli.Context, opts *options) error { - log.Infof("Installing NVIDIA container toolkit to '%v'", opts.toolkitDir) + log.Infof("Installing NVIDIA container toolkit to '%v'", opts.toolkitRoot) log.Infof("Removing existing NVIDIA container toolkit installation") - err := os.RemoveAll(opts.toolkitDir) + err := os.RemoveAll(opts.toolkitRoot) if err != nil { return fmt.Errorf("error removing toolkit directory: %v", err) } - toolkitConfigDir := filepath.Join(opts.toolkitDir, ".config", "nvidia-container-runtime") + toolkitConfigDir := filepath.Join(opts.toolkitRoot, ".config", "nvidia-container-runtime") toolkitConfigPath := filepath.Join(toolkitConfigDir, configFilename) - err = createDirectories(opts.toolkitDir, toolkitConfigDir) + err = createDirectories(opts.toolkitRoot, toolkitConfigDir) if err != nil { return fmt.Errorf("could not create required directories: %v", err) } - err = installContainerLibraries(opts.toolkitDir) + err = installContainerLibraries(opts.toolkitRoot) if err != nil { return fmt.Errorf("error installing NVIDIA container library: %v", err) } - err = installContainerRuntimes(opts.toolkitDir, opts.DriverRoot) + err = installContainerRuntimes(opts.toolkitRoot, opts.DriverRoot) if err != nil { return fmt.Errorf("error installing NVIDIA container runtime: %v", err) } - nvidiaContainerCliExecutable, err := installContainerCLI(opts.toolkitDir) + nvidiaContainerCliExecutable, err := installContainerCLI(opts.toolkitRoot) if err != nil { return fmt.Errorf("error installing NVIDIA container CLI: %v", err) } - _, err = installRuntimeHook(opts.toolkitDir, toolkitConfigPath) + _, err = installRuntimeHook(opts.toolkitRoot, toolkitConfigPath) if err != nil { return fmt.Errorf("error installing NVIDIA container runtime hook: %v", err) } @@ -198,8 +198,8 @@ func Install(cli *cli.Context, opts *options) error { // A predefined set of library candidates are considered, with the first one // resulting in success being installed to the toolkit folder. The install process // resolves the symlink for the library and copies the versioned library itself. -func installContainerLibraries(toolkitDir string) error { - log.Infof("Installing NVIDIA container library to '%v'", toolkitDir) +func installContainerLibraries(toolkitRoot string) error { + log.Infof("Installing NVIDIA container library to '%v'", toolkitRoot) libs := []string{ "libnvidia-container.so.1", @@ -207,7 +207,7 @@ func installContainerLibraries(toolkitDir string) error { } for _, l := range libs { - err := installLibrary(l, toolkitDir) + err := installLibrary(l, toolkitRoot) if err != nil { return fmt.Errorf("failed to install %s: %v", l, err) } @@ -217,15 +217,15 @@ func installContainerLibraries(toolkitDir string) error { } // installLibrary installs the specified library to the toolkit directory. -func installLibrary(libName string, toolkitDir string) error { +func installLibrary(libName string, toolkitRoot string) error { libraryPath, err := findLibrary("", libName) if err != nil { return fmt.Errorf("error locating NVIDIA container library: %v", err) } - installedLibPath, err := installFileToFolder(toolkitDir, libraryPath) + installedLibPath, err := installFileToFolder(toolkitRoot, libraryPath) if err != nil { - return fmt.Errorf("error installing %v to %v: %v", libraryPath, toolkitDir, err) + return fmt.Errorf("error installing %v to %v: %v", libraryPath, toolkitRoot, err) } log.Infof("Installed '%v' to '%v'", libraryPath, installedLibPath) @@ -233,7 +233,7 @@ func installLibrary(libName string, toolkitDir string) error { return nil } - err = installSymlink(toolkitDir, libName, installedLibPath) + err = installSymlink(toolkitRoot, libName, installedLibPath) if err != nil { return fmt.Errorf("error installing symlink for NVIDIA container library: %v", err) } @@ -297,11 +297,11 @@ func installToolkitConfig(toolkitConfigPath string, nvidiaContainerCliExecutable // installContainerCLI sets up the NVIDIA container CLI executable, copying the executable // and implementing the required wrapper -func installContainerCLI(toolkitDir string) (string, error) { +func installContainerCLI(toolkitRoot string) (string, error) { log.Infof("Installing NVIDIA container CLI from '%v'", nvidiaContainerCliSource) env := map[string]string{ - "LD_LIBRARY_PATH": toolkitDir, + "LD_LIBRARY_PATH": toolkitRoot, } e := executable{ @@ -313,7 +313,7 @@ func installContainerCLI(toolkitDir string) (string, error) { env: env, } - installedPath, err := e.install(toolkitDir) + installedPath, err := e.install(toolkitRoot) if err != nil { return "", fmt.Errorf("error installing NVIDIA container CLI: %v", err) } @@ -322,7 +322,7 @@ func installContainerCLI(toolkitDir string) (string, error) { // installRuntimeHook sets up the NVIDIA runtime hook, copying the executable // and implementing the required wrapper -func installRuntimeHook(toolkitDir string, configFilePath string) (string, error) { +func installRuntimeHook(toolkitRoot string, configFilePath string) (string, error) { log.Infof("Installing NVIDIA container runtime hook from '%v'", nvidiaContainerRuntimeHookSource) argLines := []string{ @@ -338,12 +338,12 @@ func installRuntimeHook(toolkitDir string, configFilePath string) (string, error argLines: argLines, } - installedPath, err := e.install(toolkitDir) + installedPath, err := e.install(toolkitRoot) if err != nil { return "", fmt.Errorf("error installing NVIDIA container runtime hook: %v", err) } - err = installSymlink(toolkitDir, "nvidia-container-toolkit", installedPath) + err = installSymlink(toolkitRoot, "nvidia-container-toolkit", installedPath) if err != nil { return "", fmt.Errorf("error installing symlink to NVIDIA container runtime hook: %v", err) } @@ -353,8 +353,8 @@ func installRuntimeHook(toolkitDir string, configFilePath string) (string, error // installSymlink creates a symlink in the toolkitDirectory that points to the specified target. // Note: The target is assumed to be local to the toolkit directory -func installSymlink(toolkitDir string, link string, target string) error { - symlinkPath := filepath.Join(toolkitDir, link) +func installSymlink(toolkitRoot string, link string, target string) error { + symlinkPath := filepath.Join(toolkitRoot, link) targetPath := filepath.Base(target) log.Infof("Creating symlink '%v' -> '%v'", symlinkPath, targetPath) From e848aa78134a07d81bbbbfc5ab87d28d71c08471 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 25 Jul 2022 10:13:54 +0200 Subject: [PATCH 4/7] Set toolkit root as flag Signed-off-by: Evan Lezar --- test/container/toolkit_test.sh | 4 ++-- tools/container/toolkit/toolkit.go | 25 ++++++++++++++----------- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/test/container/toolkit_test.sh b/test/container/toolkit_test.sh index 77df5fb0..84df4cac 100644 --- a/test/container/toolkit_test.sh +++ b/test/container/toolkit_test.sh @@ -23,7 +23,7 @@ testing::toolkit::install() { READLINK="greadlink" fi - testing::docker_run::toolkit::shell 'toolkit install /usr/local/nvidia/toolkit' + testing::docker_run::toolkit::shell 'toolkit install --toolkit-root=/usr/local/nvidia/toolkit' docker run --rm -v "${shared_dir}:/work" alpine sh -c "chown -R ${uid}:${gid} /work/" # Ensure toolkit dir is correctly setup @@ -66,7 +66,7 @@ testing::toolkit::install() { testing::toolkit::delete() { testing::docker_run::toolkit::shell 'mkdir -p /usr/local/nvidia/delete-toolkit' testing::docker_run::toolkit::shell 'touch /usr/local/nvidia/delete-toolkit/test.file' - testing::docker_run::toolkit::shell 'toolkit delete /usr/local/nvidia/delete-toolkit' + testing::docker_run::toolkit::shell 'toolkit delete --toolkit-root=/usr/local/nvidia/delete-toolkit' test ! -z "$(ls -A "${shared_dir}/usr/local/nvidia")" test ! -e "${shared_dir}/usr/local/nvidia/delete-toolkit" diff --git a/tools/container/toolkit/toolkit.go b/tools/container/toolkit/toolkit.go index 23d892f3..e8f7fde8 100644 --- a/tools/container/toolkit/toolkit.go +++ b/tools/container/toolkit/toolkit.go @@ -63,7 +63,7 @@ func main() { install.Usage = "Install the components of the NVIDIA container toolkit" install.ArgsUsage = "" install.Before = func(c *cli.Context) error { - return parseArgs(c, &opts) + return validateOptions(c, &opts) } install.Action = func(c *cli.Context) error { return Install(c, &opts) @@ -75,7 +75,7 @@ func main() { delete.Usage = "Delete the NVIDIA container toolkit" delete.ArgsUsage = "" delete.Before = func(c *cli.Context) error { - return parseArgs(c, &opts) + return validateOptions(c, &opts) } delete.Action = func(c *cli.Context) error { return Delete(c, &opts) @@ -111,10 +111,18 @@ func main() { Destination: &opts.ContainerCLIDebug, EnvVars: []string{"NVIDIA_CONTAINER_CLI_DEBUG"}, }, + &cli.StringFlag{ + Name: "toolkit-root", + Usage: "The directory where the NVIDIA Container toolkit is to be installed", + Required: true, + Destination: &opts.toolkitRoot, + EnvVars: []string{"TOOLKIT_ROOT"}, + }, } // Update the subcommand flags with the common subcommand flags install.Flags = append([]cli.Flag{}, flags...) + delete.Flags = append([]cli.Flag{}, flags...) // Run the top-level CLI if err := c.Run(os.Args); err != nil { @@ -122,16 +130,11 @@ func main() { } } -// parseArgs parses the command line arguments to the CLI -func parseArgs(c *cli.Context, opts *options) error { - args := c.Args() - - log.Infof("Parsing arguments: %v", args.Slice()) - if c.NArg() != 1 { - return fmt.Errorf("incorrect number of arguments") +// validateOptions checks whether the specified options are valid +func validateOptions(c *cli.Context, opts *options) error { + if opts.toolkitRoot == "" { + return fmt.Errorf("invalid --toolkit-root option: %v", opts.toolkitRoot) } - opts.toolkitRoot = args.Get(0) - log.Infof("Successfully parsed arguments") return nil } From 9c060f06ba7fa173df4806d8f914df44f681638a Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 25 Jul 2022 10:26:01 +0200 Subject: [PATCH 5/7] Remove unused TOOLKIT_ARGS / --toolkit-args Signed-off-by: Evan Lezar --- tools/container/nvidia-toolkit/run.go | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/tools/container/nvidia-toolkit/run.go b/tools/container/nvidia-toolkit/run.go index 69abc9ba..405b8e32 100644 --- a/tools/container/nvidia-toolkit/run.go +++ b/tools/container/nvidia-toolkit/run.go @@ -32,7 +32,6 @@ var signalReceived = make(chan bool, 1) var destinationArg string var noDaemonFlag bool -var toolkitArgsFlag string var runtimeFlag string var runtimeArgsFlag string @@ -44,7 +43,7 @@ func main() { c := cli.NewApp() c.Name = "nvidia-toolkit" c.Usage = "Install the nvidia-container-toolkit for use by a given runtime" - c.UsageText = "DESTINATION [-n | --no-daemon] [-t | --toolkit-args] [-r | --runtime] [-u | --runtime-args]" + c.UsageText = "DESTINATION [-n | --no-daemon] [-r | --runtime] [-u | --runtime-args]" c.Description = "DESTINATION points to the host path underneath which the nvidia-container-toolkit should be installed.\nIt will be installed at ${DESTINATION}/toolkit" c.Version = Version c.Action = Run @@ -58,14 +57,6 @@ func main() { Destination: &noDaemonFlag, EnvVars: []string{"NO_DAEMON"}, }, - &cli.StringFlag{ - Name: "toolkit-args", - Aliases: []string{"t"}, - Usage: "arguments to pass to the underlying 'toolkit' command", - Value: defaultToolkitArgs, - Destination: &toolkitArgsFlag, - EnvVars: []string{"TOOLKIT_ARGS"}, - }, &cli.StringFlag{ Name: "runtime", Aliases: []string{"r"}, @@ -221,17 +212,21 @@ func initialize() error { } func installToolkit() error { - toolkitDir := filepath.Join(destinationArg, toolkitSubDir) - log.Infof("Installing toolkit") - cmdline := fmt.Sprintf("%v install %v %v\n", toolkitCommand, toolkitArgsFlag, toolkitDir) - cmd := exec.Command("sh", "-c", cmdline) + cmdline := []string{ + toolkitCommand, + "install", + "--toolkit-root", + filepath.Join(destinationArg, toolkitSubDir), + } + + cmd := exec.Command("sh", "-c", strings.Join(cmdline, " ")) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err := cmd.Run() if err != nil { - return fmt.Errorf("error running %v command: %v", toolkitCommand, err) + return fmt.Errorf("error running %v command: %v", cmdline, err) } return nil From 90518e0ce5ee7681c7f677f8345f4f86f2269f10 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 25 Jul 2022 10:31:31 +0200 Subject: [PATCH 6/7] Allow accept-visible-devices config options to be set This change allows the * accept-nvidia-visible-devices-envvar-when-unprivileged * accept-nvidia-visible-devices-as-volume-mounts options to be set in the toolkit-container. These are controlled by command line flags or the following environment variables: * ACCEPT_NVIDIA_VISIBLE_DEVICES_ENVVAR_WHEN_UNPRIVILEGED * ACCEPT_NVIDIA_VISIBLE_DEVICES_AS_VOLUME_MOUNTS Signed-off-by: Evan Lezar --- tools/container/toolkit/toolkit.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/tools/container/toolkit/toolkit.go b/tools/container/toolkit/toolkit.go index e8f7fde8..b119eb42 100644 --- a/tools/container/toolkit/toolkit.go +++ b/tools/container/toolkit/toolkit.go @@ -45,6 +45,9 @@ type options struct { ContainerRuntimeLogLevel string ContainerCLIDebug string toolkitRoot string + + acceptNVIDIAVisibleDevicesWhenUnprivileged bool + acceptNVIDIAVisibleDevicesAsVolumeMounts bool } func main() { @@ -111,6 +114,18 @@ func main() { Destination: &opts.ContainerCLIDebug, EnvVars: []string{"NVIDIA_CONTAINER_CLI_DEBUG"}, }, + &cli.BoolFlag{ + Name: "accept-nvidia-visible-devices-envvar-when-unprivileged", + Usage: "Set the accept-nvidia-visible-devices-envvar-when-unprivileged config option", + Destination: &opts.acceptNVIDIAVisibleDevicesWhenUnprivileged, + EnvVars: []string{"ACCEPT_NVIDIA_VISIBLE_DEVICES_ENVVAR_WHEN_UNPRIVILEGED"}, + }, + &cli.BoolFlag{ + Name: "accept-nvidia-visible-devices-as-volume-mounts", + Usage: "Set the accept-nvidia-visible-devices-as-volume-mounts config option", + Destination: &opts.acceptNVIDIAVisibleDevicesWhenUnprivileged, + EnvVars: []string{"ACCEPT_NVIDIA_VISIBLE_DEVICES_AS_VOLUME_MOUNTS"}, + }, &cli.StringFlag{ Name: "toolkit-root", Usage: "The directory where the NVIDIA Container toolkit is to be installed", @@ -260,6 +275,10 @@ func installToolkitConfig(toolkitConfigPath string, nvidiaContainerCliExecutable } defer targetConfig.Close() + // Set the options in the root toml table + config.Set("accept-nvidia-visible-devices-envvar-when-unprivileged", opts.acceptNVIDIAVisibleDevicesWhenUnprivileged) + config.Set("accept-nvidia-visible-devices-as-volume-mounts", opts.acceptNVIDIAVisibleDevicesAsVolumeMounts) + nvidiaContainerCliKey := func(p string) []string { return []string{"nvidia-container-cli", p} } From 3610fe7c33a061026c15e9efb9f05005c75be829 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 27 Jul 2022 11:12:57 +0200 Subject: [PATCH 7/7] Update libnvidia-container Signed-off-by: Evan Lezar --- third_party/libnvidia-container | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/third_party/libnvidia-container b/third_party/libnvidia-container index 395fd417..ab4ac25e 160000 --- a/third_party/libnvidia-container +++ b/third_party/libnvidia-container @@ -1 +1 @@ -Subproject commit 395fd41701117121f1fd04ada01e1d7e006a37ae +Subproject commit ab4ac25ea4752ec8a01afef6c994754cf67a0796