mirror of
https://github.com/NVIDIA/nvidia-container-toolkit
synced 2025-06-26 18:18:24 +00:00
Add a hook to disable device node creation in a container
If required, this hook creates a modified params file (with ModifyDeviceFiles: 0) in a tmpfs and mounts this over /proc/driver/nvidia/params. This prevents device node creation when running tools such as nvidia-smi. In general the creation of these devices is cosmetic as a container does not have the required cgroup access for the devices. Signed-off-by: Evan Lezar <elezar@nvidia.com>
This commit is contained in:
parent
fdcd250362
commit
0c8723a93a
@ -22,6 +22,7 @@ import (
|
||||
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/chmod"
|
||||
symlinks "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/create-symlinks"
|
||||
"github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/cudacompat"
|
||||
disabledevicenodemodification "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/disable-device-node-modification"
|
||||
ldcache "github.com/NVIDIA/nvidia-container-toolkit/cmd/nvidia-cdi-hook/update-ldcache"
|
||||
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
|
||||
)
|
||||
@ -34,6 +35,7 @@ func New(logger logger.Interface) []*cli.Command {
|
||||
symlinks.NewCommand(logger),
|
||||
chmod.NewCommand(logger),
|
||||
cudacompat.NewCommand(logger),
|
||||
disabledevicenodemodification.NewCommand(logger),
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -0,0 +1,144 @@
|
||||
/**
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
**/
|
||||
|
||||
package disabledevicenodemodification
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/urfave/cli/v2"
|
||||
|
||||
"github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
|
||||
"github.com/NVIDIA/nvidia-container-toolkit/internal/oci"
|
||||
)
|
||||
|
||||
const (
|
||||
nvidiaDriverParamsPath = "/proc/driver/nvidia/params"
|
||||
)
|
||||
|
||||
type options struct {
|
||||
containerSpec string
|
||||
}
|
||||
|
||||
// NewCommand constructs an disable-device-node-modification subcommand with the specified logger
|
||||
func NewCommand(logger logger.Interface) *cli.Command {
|
||||
cfg := options{}
|
||||
|
||||
c := cli.Command{
|
||||
Name: "disable-device-node-modification",
|
||||
Usage: "Ensure that the /proc/driver/nvidia/params file present in the container does not allow device node modifications.",
|
||||
Before: func(c *cli.Context) error {
|
||||
return validateFlags(c, &cfg)
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
return run(c, &cfg)
|
||||
},
|
||||
}
|
||||
|
||||
c.Flags = []cli.Flag{
|
||||
&cli.StringFlag{
|
||||
Name: "container-spec",
|
||||
Hidden: true,
|
||||
Usage: "Specify the path to the OCI container spec. If empty or '-' the spec will be read from STDIN",
|
||||
Destination: &cfg.containerSpec,
|
||||
},
|
||||
}
|
||||
|
||||
return &c
|
||||
}
|
||||
|
||||
func validateFlags(c *cli.Context, cfg *options) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func run(_ *cli.Context, cfg *options) error {
|
||||
modifiedParamsFileContents, err := getModifiedNVIDIAParamsContents()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get modified params file contents: %w", err)
|
||||
}
|
||||
if len(modifiedParamsFileContents) == 0 {
|
||||
return nil
|
||||
}
|
||||
|
||||
s, err := oci.LoadContainerState(cfg.containerSpec)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to load container state: %w", err)
|
||||
}
|
||||
|
||||
containerRootDirPath, err := s.GetContainerRoot()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to determined container root: %w", err)
|
||||
}
|
||||
|
||||
return createParamsFileInContainer(containerRootDirPath, modifiedParamsFileContents)
|
||||
}
|
||||
|
||||
func getModifiedNVIDIAParamsContents() ([]byte, error) {
|
||||
hostNvidiaParamsFile, err := os.Open(nvidiaDriverParamsPath)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return nil, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to load params file: %w", err)
|
||||
}
|
||||
defer hostNvidiaParamsFile.Close()
|
||||
|
||||
modifiedContents, err := getModifiedParamsFileContentsFromReader(hostNvidiaParamsFile)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get modfied params file contents: %w", err)
|
||||
}
|
||||
|
||||
return modifiedContents, nil
|
||||
}
|
||||
|
||||
// getModifiedParamsFileContentsFromReader returns the contents of a modified params file from the specified reader.
|
||||
func getModifiedParamsFileContentsFromReader(r io.Reader) ([]byte, error) {
|
||||
var modified bytes.Buffer
|
||||
scanner := bufio.NewScanner(r)
|
||||
|
||||
var requiresModification bool
|
||||
for scanner.Scan() {
|
||||
line := scanner.Text()
|
||||
if strings.HasPrefix(line, "ModifyDeviceFiles: ") {
|
||||
if line == "ModifyDeviceFiles: 0" {
|
||||
return nil, nil
|
||||
}
|
||||
if line == "ModifyDeviceFiles: 1" {
|
||||
line = "ModifyDeviceFiles: 0"
|
||||
requiresModification = true
|
||||
}
|
||||
}
|
||||
if _, err := modified.WriteString(line + "\n"); err != nil {
|
||||
return nil, fmt.Errorf("failed to create output buffer: %w", err)
|
||||
}
|
||||
}
|
||||
if err := scanner.Err(); err != nil {
|
||||
return nil, fmt.Errorf("failed to read params file: %w", err)
|
||||
}
|
||||
|
||||
if !requiresModification {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
return modified.Bytes(), nil
|
||||
}
|
@ -0,0 +1,91 @@
|
||||
/**
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
**/
|
||||
|
||||
package disabledevicenodemodification
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
)
|
||||
|
||||
func TestGetModifiedParamsFileContentsFromReader(t *testing.T) {
|
||||
testCases := map[string]struct {
|
||||
contents []byte
|
||||
expectedError error
|
||||
expectedContents []byte
|
||||
}{
|
||||
"no contents": {
|
||||
contents: nil,
|
||||
expectedError: nil,
|
||||
expectedContents: nil,
|
||||
},
|
||||
"other contents are ignored": {
|
||||
contents: []byte(`# Some other content
|
||||
that we don't care about
|
||||
`),
|
||||
expectedError: nil,
|
||||
expectedContents: nil,
|
||||
},
|
||||
"already zero requires no modification": {
|
||||
contents: []byte("ModifyDeviceFiles: 0"),
|
||||
expectedError: nil,
|
||||
expectedContents: nil,
|
||||
},
|
||||
"leading spaces require no modification": {
|
||||
contents: []byte(" ModifyDeviceFiles: 1"),
|
||||
},
|
||||
"Trailing spaces require no modification": {
|
||||
contents: []byte("ModifyDeviceFiles: 1 "),
|
||||
},
|
||||
"Not 1 require no modification": {
|
||||
contents: []byte("ModifyDeviceFiles: 11"),
|
||||
},
|
||||
"single line requires modification": {
|
||||
contents: []byte("ModifyDeviceFiles: 1"),
|
||||
expectedError: nil,
|
||||
expectedContents: []byte("ModifyDeviceFiles: 0\n"),
|
||||
},
|
||||
"single line with trailing newline requires modification": {
|
||||
contents: []byte("ModifyDeviceFiles: 1\n"),
|
||||
expectedError: nil,
|
||||
expectedContents: []byte("ModifyDeviceFiles: 0\n"),
|
||||
},
|
||||
"other content is maintained": {
|
||||
contents: []byte(`ModifyDeviceFiles: 1
|
||||
other content
|
||||
that
|
||||
is maintained`),
|
||||
expectedError: nil,
|
||||
expectedContents: []byte(`ModifyDeviceFiles: 0
|
||||
other content
|
||||
that
|
||||
is maintained
|
||||
`),
|
||||
},
|
||||
}
|
||||
|
||||
for description, tc := range testCases {
|
||||
t.Run(description, func(t *testing.T) {
|
||||
contents, err := getModifiedParamsFileContentsFromReader(bytes.NewReader(tc.contents))
|
||||
require.EqualValues(t, tc.expectedError, err)
|
||||
require.EqualValues(t, string(tc.expectedContents), string(contents))
|
||||
})
|
||||
}
|
||||
|
||||
}
|
@ -0,0 +1,63 @@
|
||||
//go:build linux
|
||||
|
||||
/**
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
**/
|
||||
|
||||
package disabledevicenodemodification
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/opencontainers/runc/libcontainer/utils"
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
func createParamsFileInContainer(containerRootDirPath string, contents []byte) error {
|
||||
tmpRoot, err := os.MkdirTemp("", "nvct-empty-dir*")
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to create temp root: %w", err)
|
||||
}
|
||||
|
||||
if err := createTmpFs(tmpRoot, len(contents)); err != nil {
|
||||
return fmt.Errorf("failed to create tmpfs mount for params file: %w", err)
|
||||
}
|
||||
|
||||
modifiedParamsFile, err := os.OpenFile(filepath.Join(tmpRoot, "nvct-params"), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0444)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to open modified params file: %w", err)
|
||||
}
|
||||
defer modifiedParamsFile.Close()
|
||||
|
||||
if _, err := modifiedParamsFile.Write(contents); err != nil {
|
||||
return fmt.Errorf("failed to write temporary params file: %w", err)
|
||||
}
|
||||
|
||||
err = utils.WithProcfd(containerRootDirPath, nvidiaDriverParamsPath, func(nvidiaDriverParamsFdPath string) error {
|
||||
return unix.Mount(modifiedParamsFile.Name(), nvidiaDriverParamsFdPath, "", unix.MS_BIND|unix.MS_RDONLY|unix.MS_NODEV|unix.MS_PRIVATE|unix.MS_NOSYMFOLLOW, "")
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to mount modified params file: %w", err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func createTmpFs(target string, size int) error {
|
||||
return unix.Mount("tmpfs", target, "tmpfs", 0, fmt.Sprintf("size=%d", size))
|
||||
}
|
@ -0,0 +1,27 @@
|
||||
//go:build !linux
|
||||
// +build !linux
|
||||
|
||||
/**
|
||||
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
**/
|
||||
|
||||
package disabledevicenodemodification
|
||||
|
||||
import "fmt"
|
||||
|
||||
func createParamsFileInContainer(containerRootDirPath string, contents []byte) error {
|
||||
return fmt.Errorf("not supported")
|
||||
}
|
Loading…
Reference in New Issue
Block a user