From 8fabeed3a43f996bbb00821e3c33ab99ae86078a Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 29 Apr 2022 14:40:24 +0200 Subject: [PATCH 01/16] Update go vendoring Signed-off-by: Evan Lezar --- go.mod | 1 + go.sum | 2 + vendor/github.com/NVIDIA/go-nvml/LICENSE | 202 ++++++++++++++++++ vendor/github.com/NVIDIA/go-nvml/pkg/dl/dl.go | 82 +++++++ vendor/modules.txt | 3 + 5 files changed, 290 insertions(+) create mode 100644 vendor/github.com/NVIDIA/go-nvml/LICENSE create mode 100644 vendor/github.com/NVIDIA/go-nvml/pkg/dl/dl.go diff --git a/go.mod b/go.mod index ee9906ab..43dc8e18 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.14 require ( github.com/BurntSushi/toml v1.0.0 + github.com/NVIDIA/go-nvml v0.11.6-0 github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b github.com/containers/podman/v4 v4.0.3 github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab diff --git a/go.sum b/go.sum index d154ab26..56aa0f23 100644 --- a/go.sum +++ b/go.sum @@ -107,6 +107,8 @@ github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01 github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= +github.com/NVIDIA/go-nvml v0.11.6-0 h1:tugQzmaX84Y/6+03wZ/MAgcpfSKDkvkAWeuxFNLHmxY= +github.com/NVIDIA/go-nvml v0.11.6-0/go.mod h1:hy7HYeQy335x6nEss0Ne3PYqleRa6Ct+VKD9RQ4nyFs= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.0.1/go.mod h1:xsIw86fROiiwelg+jB2uM9PiKihMMmUx/1V+TNhjQvM= diff --git a/vendor/github.com/NVIDIA/go-nvml/LICENSE b/vendor/github.com/NVIDIA/go-nvml/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/vendor/github.com/NVIDIA/go-nvml/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/NVIDIA/go-nvml/pkg/dl/dl.go b/vendor/github.com/NVIDIA/go-nvml/pkg/dl/dl.go new file mode 100644 index 00000000..21a02091 --- /dev/null +++ b/vendor/github.com/NVIDIA/go-nvml/pkg/dl/dl.go @@ -0,0 +1,82 @@ +// Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dl + +import ( + "fmt" + "unsafe" +) + +// #cgo LDFLAGS: -ldl +// #include +// #include +import "C" + +const ( + RTLD_LAZY = C.RTLD_LAZY + RTLD_NOW = C.RTLD_NOW + RTLD_GLOBAL = C.RTLD_GLOBAL + RTLD_LOCAL = C.RTLD_LOCAL + RTLD_NODELETE = C.RTLD_NODELETE + RTLD_NOLOAD = C.RTLD_NOLOAD + RTLD_DEEPBIND = C.RTLD_DEEPBIND +) + +type DynamicLibrary struct{ + Name string + Flags int + handle unsafe.Pointer +} + +func New(name string, flags int) *DynamicLibrary { + return &DynamicLibrary{ + Name: name, + Flags: flags, + handle: nil, + } +} + +func (dl *DynamicLibrary) Open() error { + name := C.CString(dl.Name) + defer C.free(unsafe.Pointer(name)) + + handle := C.dlopen(name, C.int(dl.Flags)) + if handle == C.NULL { + return fmt.Errorf("%s", C.GoString(C.dlerror())) + } + dl.handle = handle + return nil +} + +func (dl *DynamicLibrary) Close() error { + err := C.dlclose(dl.handle) + if err != 0 { + return fmt.Errorf("%s", C.GoString(C.dlerror())) + } + return nil +} + +func (dl *DynamicLibrary) Lookup(symbol string) error { + sym := C.CString(symbol) + defer C.free(unsafe.Pointer(sym)) + + C.dlerror() // Clear out any previous errors + C.dlsym(dl.handle, sym) + err := C.dlerror() + if unsafe.Pointer(err) == C.NULL { + return nil + } + return fmt.Errorf("%s", C.GoString(err)) +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 670896fd..620f6f6a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -2,6 +2,9 @@ ## explicit github.com/BurntSushi/toml github.com/BurntSushi/toml/internal +# github.com/NVIDIA/go-nvml v0.11.6-0 +## explicit +github.com/NVIDIA/go-nvml/pkg/dl # github.com/blang/semver v3.5.1+incompatible github.com/blang/semver # github.com/container-orchestrated-devices/container-device-interface v0.3.1-0.20220224133719-e5457123010b From fd06c7a00bfcd2c3954c33c6ba54c0481bb3560e Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 2 May 2022 15:41:40 +0200 Subject: [PATCH 02/16] Bump golang version to 1.17.8 Signed-off-by: Evan Lezar --- versions.mk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/versions.mk b/versions.mk index e59256c0..e7e3311a 100644 --- a/versions.mk +++ b/versions.mk @@ -25,4 +25,4 @@ NVIDIA_CONTAINER_RUNTIME_VERSION := 3.10.0 LIBNVIDIA_CONTAINER0_VERSION := 0.10.0+jetpack CUDA_VERSION := 11.6.0 -GOLANG_VERSION := 1.16.4 +GOLANG_VERSION := 1.17.8 From f4d87e6912eec9f52fd368528ca80097dcc9d6d7 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 2 May 2022 15:42:07 +0200 Subject: [PATCH 03/16] Use go install to install go development tools Signed-off-by: Evan Lezar --- docker/Dockerfile.devel | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/Dockerfile.devel b/docker/Dockerfile.devel index 20e0447f..cb70df4d 100644 --- a/docker/Dockerfile.devel +++ b/docker/Dockerfile.devel @@ -14,7 +14,7 @@ ARG GOLANG_VERSION=x.x.x FROM golang:${GOLANG_VERSION} -RUN go get -u golang.org/x/lint/golint -RUN go get -u github.com/matryer/moq -RUN go get -u github.com/gordonklaus/ineffassign -RUN go get -u github.com/client9/misspell/cmd/misspell \ No newline at end of file +RUN go install golang.org/x/lint/golint@latest +RUN go install github.com/matryer/moq@latest +RUN go install github.com/gordonklaus/ineffassign@latest +RUN go install github.com/client9/misspell/cmd/misspell@latest From 2e319b5b0813d3fbda7fbbc554f0013245830d3f Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 2 May 2022 18:00:29 +0200 Subject: [PATCH 04/16] Add gcc for Amazonlinux builds Signed-off-by: Evan Lezar --- docker/Dockerfile.amazonlinux | 1 + 1 file changed, 1 insertion(+) diff --git a/docker/Dockerfile.amazonlinux b/docker/Dockerfile.amazonlinux index 14d6d36f..e7a49fcc 100644 --- a/docker/Dockerfile.amazonlinux +++ b/docker/Dockerfile.amazonlinux @@ -3,6 +3,7 @@ FROM ${BASEIMAGE} RUN yum install -y \ ca-certificates \ + gcc \ wget \ git \ rpm-build \ From 8f0e1906c29e78133d60102e47ef48434a1e85b6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 26 Apr 2022 12:07:01 +0200 Subject: [PATCH 05/16] Add CUDA image abstraction This change adds a CUDA image abstraction that encapsulates the queries performed on a container image (e.g. envvars) to check certain CUDA properties. Signed-off-by: Evan Lezar --- internal/config/image/cuda_image.go | 143 +++++++++++++++++++++++ internal/config/image/cuda_image_test.go | 71 +++++++++++ 2 files changed, 214 insertions(+) create mode 100644 internal/config/image/cuda_image.go create mode 100644 internal/config/image/cuda_image_test.go diff --git a/internal/config/image/cuda_image.go b/internal/config/image/cuda_image.go new file mode 100644 index 00000000..2e986773 --- /dev/null +++ b/internal/config/image/cuda_image.go @@ -0,0 +1,143 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package image + +import ( + "fmt" + "strconv" + "strings" + + "github.com/opencontainers/runtime-spec/specs-go" + "golang.org/x/mod/semver" +) + +const ( + envCUDAVersion = "CUDA_VERSION" + envNVRequirePrefix = "NVIDIA_REQUIRE_" + envNVRequireCUDA = envNVRequirePrefix + "CUDA" + envNVDisableRequire = "NVIDIA_DISABLE_REQUIRE" +) + +// CUDA represents a CUDA image that can be used for GPU computing. This wraps +// a map of environment variable to values that can be used to perform lookups +// such as requirements. +type CUDA map[string]string + +// NewCUDAImageFromSpec creates a CUDA image from the input OCI runtime spec. +// The process environment is read (if present) to construc the CUDA Image. +func NewCUDAImageFromSpec(spec *specs.Spec) (CUDA, error) { + if spec == nil || spec.Process == nil { + return NewCUDAImageFromEnv(nil) + } + + return NewCUDAImageFromEnv(spec.Process.Env) +} + +// NewCUDAImageFromEnv creates a CUDA image from the input environment. The environment +// is a list of strings of the form ENVAR=VALUE. +func NewCUDAImageFromEnv(env []string) (CUDA, error) { + c := make(CUDA) + + for _, e := range env { + parts := strings.SplitN(e, "=", 2) + if len(parts) != 2 { + return nil, fmt.Errorf("invalid environment variable: %v", e) + } + c[parts[0]] = parts[1] + } + + return c, nil +} + +// IsLegacy returns whether the associated CUDA image is a "legacy" image. An +// image is considered legacy if it has a CUDA_VERSION environment variable defined +// and no NVIDIA_REQUIRE_CUDA environment variable defined. +func (i CUDA) IsLegacy() bool { + legacyCudaVersion := i[envCUDAVersion] + cudaRequire := i[envNVRequireCUDA] + return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0 +} + +// GetRequirements returns the requirements fomr all NVIDIA_REQUIRE_ environment +// variables. +func (i CUDA) GetRequirements() ([]string, error) { + // TODO: We need not process this if disable require is set, but this will be done + // in a single follow-up to ensure that the behavioural change is accurately captured. + // if i.HasDisableRequire() { + // return nil, nil + // } + + // All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli + var requirements []string + for name, value := range i { + if strings.HasPrefix(name, envNVRequirePrefix) { + requirements = append(requirements, value) + } + } + if i.IsLegacy() { + v, err := i.legacyVersion() + if err != nil { + return nil, fmt.Errorf("failed to get version: %v", err) + } + cudaRequire := fmt.Sprintf("cuda>=%s", v) + requirements = append(requirements, cudaRequire) + } + return requirements, nil +} + +// HasDisableRequire checks for the value of the NVIDIA_DISABLE_REQUIRE. If set +// to a valid (true) boolean value this can be used to disable the requirement checks +func (i CUDA) HasDisableRequire() bool { + if disable, exists := i[envNVDisableRequire]; exists { + // i.logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", disable) + d, _ := strconv.ParseBool(disable) + return d + } + + return false +} + +func (i CUDA) legacyVersion() (string, error) { + majorMinor, err := parseMajorMinorVersion(i[envCUDAVersion]) + if err != nil { + return "", fmt.Errorf("invalid CUDA version: %v", err) + } + + return majorMinor, nil +} + +func parseMajorMinorVersion(version string) (string, error) { + vVersion := "v" + strings.TrimPrefix(version, "v") + + if !semver.IsValid(vVersion) { + return "", fmt.Errorf("invalid version string") + } + + majorMinor := strings.TrimPrefix(semver.MajorMinor(vVersion), "v") + parts := strings.Split(majorMinor, ".") + + var err error + _, err = strconv.ParseUint(parts[0], 10, 32) + if err != nil { + return "", fmt.Errorf("invalid major version") + } + _, err = strconv.ParseUint(parts[1], 10, 32) + if err != nil { + return "", fmt.Errorf("invalid minor version") + } + return majorMinor, nil +} diff --git a/internal/config/image/cuda_image_test.go b/internal/config/image/cuda_image_test.go new file mode 100644 index 00000000..71d11cd2 --- /dev/null +++ b/internal/config/image/cuda_image_test.go @@ -0,0 +1,71 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package image + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestParseMajorMinorVersionValid(t *testing.T) { + var tests = []struct { + version string + expected string + }{ + {"0", "0.0"}, + {"8", "8.0"}, + {"7.5", "7.5"}, + {"9.0.116", "9.0"}, + {"4294967295.4294967295.4294967295", "4294967295.4294967295"}, + {"v11.6", "11.6"}, + } + for _, c := range tests { + t.Run(c.version, func(t *testing.T) { + version, err := parseMajorMinorVersion(c.version) + + require.NoError(t, err) + require.Equal(t, c.expected, version) + }) + } +} + +func TestParseMajorMinorVersionInvalid(t *testing.T) { + var tests = []string{ + "foo", + "foo.5.10", + "9.0.116.50", + "9.0.116foo", + "7.foo", + "9.0.bar", + "9.4294967296", + "9.0.116.", + "9..0", + "9.", + ".5.10", + "-9", + "+9", + "-9.1.116", + "-9.-1.-116", + } + for _, c := range tests { + t.Run(c, func(t *testing.T) { + _, err := parseMajorMinorVersion(c) + require.Error(t, err) + }) + } +} From 50cf07e4cdb9bb70e07fc3cff037bfd42c30b85a Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Tue, 26 Apr 2022 13:44:44 +0200 Subject: [PATCH 06/16] Use CUDA image abstraction for runtime hook Signed-off-by: Evan Lezar --- .../container_config.go | 93 ++++--------------- cmd/nvidia-container-toolkit/hook_test.go | 45 --------- 2 files changed, 19 insertions(+), 119 deletions(-) diff --git a/cmd/nvidia-container-toolkit/container_config.go b/cmd/nvidia-container-toolkit/container_config.go index ee3492b3..5920f471 100644 --- a/cmd/nvidia-container-toolkit/container_config.go +++ b/cmd/nvidia-container-toolkit/container_config.go @@ -7,9 +7,9 @@ import ( "os" "path" "path/filepath" - "strconv" "strings" + "github.com/NVIDIA/nvidia-container-toolkit/internal/config/image" "golang.org/x/mod/semver" ) @@ -104,45 +104,6 @@ type HookState struct { BundlePath string `json:"bundlePath"` } -func parseCudaVersion(cudaVersion string) (uint32, uint32) { - major, minor, err := parseMajorMinorVersion(cudaVersion) - if err != nil { - log.Panicln("invalid CUDA Version", cudaVersion, err) - } - return major, minor -} - -func parseMajorMinorVersion(version string) (uint32, uint32, error) { - if !semver.IsValid("v" + version) { - return 0, 0, fmt.Errorf("invalid version string") - } - - majorMinor := strings.TrimPrefix(semver.MajorMinor("v"+version), "v") - parts := strings.Split(majorMinor, ".") - - major, err := strconv.ParseUint(parts[0], 10, 32) - if err != nil { - return 0, 0, fmt.Errorf("invalid major version") - } - minor, err := strconv.ParseUint(parts[1], 10, 32) - if err != nil { - return 0, 0, fmt.Errorf("invalid minor version") - } - return uint32(major), uint32(minor), nil -} - -func getEnvMap(e []string) (m map[string]string) { - m = make(map[string]string) - for _, s := range e { - p := strings.SplitN(s, "=", 2) - if len(p) != 2 { - log.Panicln("environment error") - } - m[p[0]] = p[1] - } - return -} - func loadSpec(path string) (spec *Spec) { f, err := os.Open(path) if err != nil { @@ -204,12 +165,6 @@ func isPrivileged(s *Spec) bool { return false } -func isLegacyCUDAImage(env map[string]string) bool { - legacyCudaVersion := env[envCUDAVersion] - cudaRequire := env[envNVRequireCUDA] - return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0 -} - func getDevicesFromEnvvar(env map[string]string, legacyImage bool) *string { // Build a list of envvars to consider. envVars := []string{envNVVisibleDevices} @@ -348,27 +303,11 @@ func getDriverCapabilities(env map[string]string, supportedDriverCapabilities Dr return capabilities } -func getRequirements(env map[string]string, legacyImage bool) []string { - // All variables with the "NVIDIA_REQUIRE_" prefix are passed to nvidia-container-cli - var requirements []string - for name, value := range env { - if strings.HasPrefix(name, envNVRequirePrefix) { - requirements = append(requirements, value) - } - } - if legacyImage { - vmaj, vmin := parseCudaVersion(env[envCUDAVersion]) - cudaRequire := fmt.Sprintf("cuda>=%d.%d", vmaj, vmin) - requirements = append(requirements, cudaRequire) - } - return requirements -} - -func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mount, privileged bool) *nvidiaConfig { - legacyImage := isLegacyCUDAImage(env) +func getNvidiaConfig(hookConfig *HookConfig, image image.CUDA, mounts []Mount, privileged bool) *nvidiaConfig { + legacyImage := image.IsLegacy() var devices string - if d := getDevices(hookConfig, env, mounts, privileged, legacyImage); d != nil { + if d := getDevices(hookConfig, image, mounts, privileged, legacyImage); d != nil { devices = *d } else { // 'nil' devices means this is not a GPU container. @@ -376,7 +315,7 @@ func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mou } var migConfigDevices string - if d := getMigConfigDevices(env); d != nil { + if d := getMigConfigDevices(image); d != nil { migConfigDevices = *d } if !privileged && migConfigDevices != "" { @@ -384,19 +323,21 @@ func getNvidiaConfig(hookConfig *HookConfig, env map[string]string, mounts []Mou } var migMonitorDevices string - if d := getMigMonitorDevices(env); d != nil { + if d := getMigMonitorDevices(image); d != nil { migMonitorDevices = *d } if !privileged && migMonitorDevices != "" { log.Panicln("cannot set MIG_MONITOR_DEVICES in non privileged container") } - driverCapabilities := getDriverCapabilities(env, hookConfig.SupportedDriverCapabilities, legacyImage).String() + driverCapabilities := getDriverCapabilities(image, hookConfig.SupportedDriverCapabilities, legacyImage).String() - requirements := getRequirements(env, legacyImage) + requirements, err := image.GetRequirements() + if err != nil { + log.Panicln("failed to get requirements", err) + } - // Don't fail on invalid values. - disableRequire, _ := strconv.ParseBool(env[envNVDisableRequire]) + disableRequire := image.HasDisableRequire() return &nvidiaConfig{ Devices: devices, @@ -422,13 +363,17 @@ func getContainerConfig(hook HookConfig) (config containerConfig) { s := loadSpec(path.Join(b, "config.json")) - env := getEnvMap(s.Process.Env) + image, err := image.NewCUDAImageFromEnv(s.Process.Env) + if err != nil { + log.Panicln(err) + } + privileged := isPrivileged(s) envSwarmGPU = hook.SwarmResource return containerConfig{ Pid: h.Pid, Rootfs: s.Root.Path, - Env: env, - Nvidia: getNvidiaConfig(&hook, env, s.Mounts, privileged), + Env: image, + Nvidia: getNvidiaConfig(&hook, image, s.Mounts, privileged), } } diff --git a/cmd/nvidia-container-toolkit/hook_test.go b/cmd/nvidia-container-toolkit/hook_test.go index 0e955737..d5449bab 100644 --- a/cmd/nvidia-container-toolkit/hook_test.go +++ b/cmd/nvidia-container-toolkit/hook_test.go @@ -7,51 +7,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestParseCudaVersionValid(t *testing.T) { - var tests = []struct { - version string - expected [2]uint32 - }{ - {"0", [2]uint32{0, 0}}, - {"8", [2]uint32{8, 0}}, - {"7.5", [2]uint32{7, 5}}, - {"9.0.116", [2]uint32{9, 0}}, - {"4294967295.4294967295.4294967295", [2]uint32{4294967295, 4294967295}}, - } - for i, c := range tests { - vmaj, vmin := parseCudaVersion(c.version) - - version := [2]uint32{vmaj, vmin} - - require.Equal(t, c.expected, version, "%d: %v", i, c) - } -} - -func TestParseCudaVersionInvalid(t *testing.T) { - var tests = []string{ - "foo", - "foo.5.10", - "9.0.116.50", - "9.0.116foo", - "7.foo", - "9.0.bar", - "9.4294967296", - "9.0.116.", - "9..0", - "9.", - ".5.10", - "-9", - "+9", - "-9.1.116", - "-9.-1.-116", - } - for _, c := range tests { - require.Panics(t, func() { - parseCudaVersion(c) - }, "parseCudaVersion(%v)", c) - } -} - func TestIsPrivileged(t *testing.T) { var tests = []struct { spec string From a672713dba9fe4cd1671a02c614f57ed86498c8d Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 29 Apr 2022 14:38:13 +0200 Subject: [PATCH 07/16] Add basic CUDA wrapper Signed-off-by: Evan Lezar --- internal/cuda/cuda.go | 85 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 85 insertions(+) create mode 100644 internal/cuda/cuda.go diff --git a/internal/cuda/cuda.go b/internal/cuda/cuda.go new file mode 100644 index 00000000..6224de3e --- /dev/null +++ b/internal/cuda/cuda.go @@ -0,0 +1,85 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package cuda + +import ( + "fmt" + + "github.com/NVIDIA/go-nvml/pkg/dl" +) + +/* +#cgo LDFLAGS: -Wl,--unresolved-symbols=ignore-in-object-files + +#ifdef _WIN32 +#define CUDAAPI __stdcall +#else +#define CUDAAPI +#endif + +typedef enum cudaError_enum { + CUDA_SUCCESS = 0 +} CUresult; + +CUresult CUDAAPI cuDriverGetVersion(int *driverVersion); +*/ +import "C" + +const ( + libraryName = "libcuda.so.1" + libraryLoadFlags = dl.RTLD_LAZY | dl.RTLD_GLOBAL +) + +// cuda stores a reference the cuda dynamic library +var lib *dl.DynamicLibrary + +// Version returns the CUDA version of the driver as a string or an error if this +// cannot be determined. +func Version() (string, error) { + lib, err := load() + if err != nil { + return "", err + } + defer lib.Close() + + if err := lib.Lookup("cuDriverGetVersion"); err != nil { + return "", fmt.Errorf("failed to lookup symbol: %v", err) + } + + var version C.int + if result := C.cuDriverGetVersion(&version); result != C.CUDA_SUCCESS { + return "", fmt.Errorf("failed to get CUDA version: result=%v", result) + } + + major := version / 1000 + minor := version % 100 / 10 + + return fmt.Sprintf("%d.%d", major, minor), nil +} + +func load() (*dl.DynamicLibrary, error) { + lib := dl.New(libraryName, libraryLoadFlags) + if lib == nil { + return nil, fmt.Errorf("error instantiating DynamicLibrary for CUDA") + } + err := lib.Open() + if err != nil { + return nil, fmt.Errorf("error opening DynamicLibrary for CUDA: %v", err) + } + + return lib, nil +} From 5d7b3a4a9667987a21671ead6a2ac6bd324175d3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 2 May 2022 10:27:53 +0200 Subject: [PATCH 08/16] Return raw spec from Spec.Load Signed-off-by: Evan Lezar --- .../modifier/experimental.go | 3 ++- internal/oci/spec.go | 2 +- internal/oci/spec_file.go | 8 ++++---- internal/oci/spec_memory.go | 4 ++-- internal/oci/spec_mock.go | 12 +++++++----- internal/runtime/runtime_modifier.go | 2 +- 6 files changed, 17 insertions(+), 14 deletions(-) diff --git a/cmd/nvidia-container-runtime/modifier/experimental.go b/cmd/nvidia-container-runtime/modifier/experimental.go index f8e927c5..2a048369 100644 --- a/cmd/nvidia-container-runtime/modifier/experimental.go +++ b/cmd/nvidia-container-runtime/modifier/experimental.go @@ -46,7 +46,8 @@ const ( // NewExperimentalModifier creates a modifier that applies the experimental // modications to an OCI spec if required by the runtime wrapper. func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) { - if err := ociSpec.Load(); err != nil { + _, err := ociSpec.Load() + if err != nil { return nil, fmt.Errorf("failed to load OCI spec: %v", err) } diff --git a/internal/oci/spec.go b/internal/oci/spec.go index ba7e7cee..407665c4 100644 --- a/internal/oci/spec.go +++ b/internal/oci/spec.go @@ -33,7 +33,7 @@ type SpecModifier interface { //go:generate moq -stub -out spec_mock.go . Spec // Spec defines the operations to be performed on an OCI specification type Spec interface { - Load() error + Load() (*specs.Spec, error) Flush() error Modify(SpecModifier) error LookupEnv(string) (string, bool) diff --git a/internal/oci/spec_file.go b/internal/oci/spec_file.go index 3465652d..7fba130d 100644 --- a/internal/oci/spec_file.go +++ b/internal/oci/spec_file.go @@ -45,19 +45,19 @@ func NewFileSpec(filepath string) Spec { // Load reads the contents of an OCI spec from file to be referenced internally. // The file is opened "read-only" -func (s *fileSpec) Load() error { +func (s *fileSpec) Load() (*specs.Spec, error) { specFile, err := os.Open(s.path) if err != nil { - return fmt.Errorf("error opening OCI specification file: %v", err) + return nil, fmt.Errorf("error opening OCI specification file: %v", err) } defer specFile.Close() spec, err := LoadFrom(specFile) if err != nil { - return fmt.Errorf("error loading OCI specification from file: %v", err) + return nil, fmt.Errorf("error loading OCI specification from file: %v", err) } s.Spec = spec - return nil + return s.Spec, nil } // LoadFrom reads the contents of the OCI spec from the specified io.Reader. diff --git a/internal/oci/spec_memory.go b/internal/oci/spec_memory.go index ce94447e..478db5a2 100644 --- a/internal/oci/spec_memory.go +++ b/internal/oci/spec_memory.go @@ -37,8 +37,8 @@ func NewMemorySpec(spec *specs.Spec) Spec { } // Load is a no-op for the memorySpec spec -func (s *memorySpec) Load() error { - return nil +func (s *memorySpec) Load() (*specs.Spec, error) { + return s.Spec, nil } // Flush is a no-op for the memorySpec spec diff --git a/internal/oci/spec_mock.go b/internal/oci/spec_mock.go index 1656552c..79676590 100644 --- a/internal/oci/spec_mock.go +++ b/internal/oci/spec_mock.go @@ -4,6 +4,7 @@ package oci import ( + "github.com/opencontainers/runtime-spec/specs-go" "sync" ) @@ -20,7 +21,7 @@ var _ Spec = &SpecMock{} // FlushFunc: func() error { // panic("mock out the Flush method") // }, -// LoadFunc: func() error { +// LoadFunc: func() (*specs.Spec, error) { // panic("mock out the Load method") // }, // LookupEnvFunc: func(s string) (string, bool) { @@ -40,7 +41,7 @@ type SpecMock struct { FlushFunc func() error // LoadFunc mocks the Load method. - LoadFunc func() error + LoadFunc func() (*specs.Spec, error) // LookupEnvFunc mocks the LookupEnv method. LookupEnvFunc func(s string) (string, bool) @@ -103,7 +104,7 @@ func (mock *SpecMock) FlushCalls() []struct { } // Load calls LoadFunc. -func (mock *SpecMock) Load() error { +func (mock *SpecMock) Load() (*specs.Spec, error) { callInfo := struct { }{} mock.lockLoad.Lock() @@ -111,9 +112,10 @@ func (mock *SpecMock) Load() error { mock.lockLoad.Unlock() if mock.LoadFunc == nil { var ( - errOut error + specOut *specs.Spec + errOut error ) - return errOut + return specOut, errOut } return mock.LoadFunc() } diff --git a/internal/runtime/runtime_modifier.go b/internal/runtime/runtime_modifier.go index f385d9c3..d11fda82 100644 --- a/internal/runtime/runtime_modifier.go +++ b/internal/runtime/runtime_modifier.go @@ -68,7 +68,7 @@ func (r *modifyingRuntimeWrapper) Exec(args []string) error { // modify loads, modifies, and flushes the OCI specification using the defined Modifier func (r *modifyingRuntimeWrapper) modify() error { - err := r.ociSpec.Load() + _, err := r.ociSpec.Load() if err != nil { return fmt.Errorf("error loading OCI specification for modification: %v", err) } From 583793b7ae57a342f0e1f435a88e5fc5b63e755b Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 21 Apr 2022 17:25:43 +0200 Subject: [PATCH 09/16] Add processing for requirements and constraints This change adds a Requirements abstraction that can be used to check an images' NVIDIA_REQUIRE_* envvars against the host properties such as CUDA version or architecture. Signed-off-by: Evan Lezar --- internal/requirements/constants.go | 25 ++ internal/requirements/constraints/binary.go | 76 ++++++ .../requirements/constraints/constants.go | 51 ++++ .../constraints/constraint_mock.go | 108 ++++++++ .../requirements/constraints/constraints.go | 24 ++ .../constraints/constraints_test.go | 17 ++ internal/requirements/constraints/factory.go | 143 +++++++++++ .../requirements/constraints/factory_test.go | 187 ++++++++++++++ internal/requirements/constraints/logical.go | 91 +++++++ .../requirements/constraints/logical_test.go | 152 +++++++++++ internal/requirements/constraints/property.go | 129 ++++++++++ .../requirements/constraints/property_mock.go | 241 ++++++++++++++++++ internal/requirements/requirements.go | 69 +++++ 13 files changed, 1313 insertions(+) create mode 100644 internal/requirements/constants.go create mode 100644 internal/requirements/constraints/binary.go create mode 100644 internal/requirements/constraints/constants.go create mode 100644 internal/requirements/constraints/constraint_mock.go create mode 100644 internal/requirements/constraints/constraints.go create mode 100644 internal/requirements/constraints/constraints_test.go create mode 100644 internal/requirements/constraints/factory.go create mode 100644 internal/requirements/constraints/factory_test.go create mode 100644 internal/requirements/constraints/logical.go create mode 100644 internal/requirements/constraints/logical_test.go create mode 100644 internal/requirements/constraints/property.go create mode 100644 internal/requirements/constraints/property_mock.go create mode 100644 internal/requirements/requirements.go diff --git a/internal/requirements/constants.go b/internal/requirements/constants.go new file mode 100644 index 00000000..e9ea7077 --- /dev/null +++ b/internal/requirements/constants.go @@ -0,0 +1,25 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package requirements + +// A list of supported requirements / properties +const ( + ARCH = "arch" + BRAND = "brand" + CUDA = "cuda" + DRIVER = "driver" +) diff --git a/internal/requirements/constraints/binary.go b/internal/requirements/constraints/binary.go new file mode 100644 index 00000000..5ca37408 --- /dev/null +++ b/internal/requirements/constraints/binary.go @@ -0,0 +1,76 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints + +import ( + "fmt" +) + +// binary represents a binary operation. This can be used to compare a specified +// property to a value +type binary struct { + left Property + operator string + right string +} + +// String returns the string representation of the binary comparator +func (c binary) String() string { + return fmt.Sprintf("%v%v%v", c.left.Name(), c.operator, c.right) +} + +// Assert compares the property to the required value using the supplied comparator +func (c binary) Assert() error { + satisfied, err := c.eval() + if err != nil { + return err + } + if satisfied { + return nil + } + + // error_setx(err, "unsatisfied condition: %s, please update your driver to a newer version, or use an earlier cuda container", predicate_format); + return fmt.Errorf("unsatisfied condition: %v (%v)", c.String(), c.left.String()) +} + +func (c binary) eval() (bool, error) { + if c.left == nil { + return true, nil + } + + compare, err := c.left.CompareTo(c.right) + if err != nil { + return false, err + } + + switch string(c.operator) { + case equal: + return compare == 0, nil + case notEqual: + return compare != 0, nil + case less: + return compare < 0, nil + case lessEqual: + return compare <= 0, nil + case greater: + return compare > 0, nil + case greaterEqual: + return compare >= 0, nil + } + + return false, fmt.Errorf("invalid operator %v", c.operator) +} diff --git a/internal/requirements/constraints/constants.go b/internal/requirements/constraints/constants.go new file mode 100644 index 00000000..9e515214 --- /dev/null +++ b/internal/requirements/constraints/constants.go @@ -0,0 +1,51 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints + +import "fmt" + +const ( + equal = "=" + notEqual = "!=" + less = "<" + lessEqual = "<=" + greater = ">" + greaterEqual = ">=" +) + +// always is a constraint that is always met +type always struct{} + +func (c always) Assert() error { + return nil +} + +func (c always) String() string { + return "true" +} + +// invalid is an invalid constraint and can never be met +type invalid string + +func (c invalid) Assert() error { + return fmt.Errorf("invalid constraint: %v", c.String()) +} + +// String returns the string representation of the contraint +func (c invalid) String() string { + return string(c) +} diff --git a/internal/requirements/constraints/constraint_mock.go b/internal/requirements/constraints/constraint_mock.go new file mode 100644 index 00000000..08cd46a1 --- /dev/null +++ b/internal/requirements/constraints/constraint_mock.go @@ -0,0 +1,108 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package constraints + +import ( + "sync" +) + +// Ensure, that ConstraintMock does implement Constraint. +// If this is not the case, regenerate this file with moq. +var _ Constraint = &ConstraintMock{} + +// ConstraintMock is a mock implementation of Constraint. +// +// func TestSomethingThatUsesConstraint(t *testing.T) { +// +// // make and configure a mocked Constraint +// mockedConstraint := &ConstraintMock{ +// AssertFunc: func() error { +// panic("mock out the Assert method") +// }, +// StringFunc: func() string { +// panic("mock out the String method") +// }, +// } +// +// // use mockedConstraint in code that requires Constraint +// // and then make assertions. +// +// } +type ConstraintMock struct { + // AssertFunc mocks the Assert method. + AssertFunc func() error + + // StringFunc mocks the String method. + StringFunc func() string + + // calls tracks calls to the methods. + calls struct { + // Assert holds details about calls to the Assert method. + Assert []struct { + } + // String holds details about calls to the String method. + String []struct { + } + } + lockAssert sync.RWMutex + lockString sync.RWMutex +} + +// Assert calls AssertFunc. +func (mock *ConstraintMock) Assert() error { + callInfo := struct { + }{} + mock.lockAssert.Lock() + mock.calls.Assert = append(mock.calls.Assert, callInfo) + mock.lockAssert.Unlock() + if mock.AssertFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.AssertFunc() +} + +// AssertCalls gets all the calls that were made to Assert. +// Check the length with: +// len(mockedConstraint.AssertCalls()) +func (mock *ConstraintMock) AssertCalls() []struct { +} { + var calls []struct { + } + mock.lockAssert.RLock() + calls = mock.calls.Assert + mock.lockAssert.RUnlock() + return calls +} + +// String calls StringFunc. +func (mock *ConstraintMock) String() string { + callInfo := struct { + }{} + mock.lockString.Lock() + mock.calls.String = append(mock.calls.String, callInfo) + mock.lockString.Unlock() + if mock.StringFunc == nil { + var ( + sOut string + ) + return sOut + } + return mock.StringFunc() +} + +// StringCalls gets all the calls that were made to String. +// Check the length with: +// len(mockedConstraint.StringCalls()) +func (mock *ConstraintMock) StringCalls() []struct { +} { + var calls []struct { + } + mock.lockString.RLock() + calls = mock.calls.String + mock.lockString.RUnlock() + return calls +} diff --git a/internal/requirements/constraints/constraints.go b/internal/requirements/constraints/constraints.go new file mode 100644 index 00000000..48e5310a --- /dev/null +++ b/internal/requirements/constraints/constraints.go @@ -0,0 +1,24 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints + +//go:generate moq -stub -out constraint_mock.go . Constraint +// Constraint represents a constraint that is to be evaluated +type Constraint interface { + String() string + Assert() error +} diff --git a/internal/requirements/constraints/constraints_test.go b/internal/requirements/constraints/constraints_test.go new file mode 100644 index 00000000..ad6a172a --- /dev/null +++ b/internal/requirements/constraints/constraints_test.go @@ -0,0 +1,17 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints diff --git a/internal/requirements/constraints/factory.go b/internal/requirements/constraints/factory.go new file mode 100644 index 00000000..29227c66 --- /dev/null +++ b/internal/requirements/constraints/factory.go @@ -0,0 +1,143 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints + +import ( + "fmt" + "strings" + + "github.com/sirupsen/logrus" +) + +type factory struct { + logger *logrus.Logger + properties map[string]Property +} + +// New creates a new constraint for the supplied requirements and properties +func New(logger *logrus.Logger, requirements []string, properties map[string]Property) (Constraint, error) { + if len(requirements) == 0 { + return &always{}, nil + } + + f := factory{ + logger: logger, + properties: properties, + } + + var constraints []Constraint + for _, r := range requirements { + c, err := f.newConstraintFromRequirement(r) + if err != nil { + return nil, err + } + if c == nil { + continue + } + constraints = append(constraints, c) + } + + return AND(constraints), nil +} + +// newConstraintFromRequirement takes a requirement string and generates +// the associated constraint(s). Invalid constraints are ignored. +// Each requirement can consist of multiple constraints, with space-separated constraints being ORed +// together and comma-separated constraints being ANDed together. +func (r factory) newConstraintFromRequirement(requirement string) (Constraint, error) { + const ( + orSeparator = " " + andSeparator = "," + ) + if strings.TrimSpace(requirement) == "" { + return nil, nil + } + + var terms []Constraint + for _, term := range strings.Split(requirement, orSeparator) { + var factors []Constraint + for _, factor := range strings.Split(term, andSeparator) { + f, err := r.parse(factor) + if err != nil { + return nil, err + } + if f == nil { + r.logger.Debugf("Skipping unsupported constraint: %v", factor) + continue + } + factors = append(factors, f) + } + + if len(factors) == 0 { + continue + } + + if len(factors) == 1 { + terms = append(terms, factors[0]) + } else { + terms = append(terms, and(factors)) + } + } + + return OR(terms), nil +} + +// parse constructs a constraint from the specified string. +// The string is expected to be of the form [PROPERTY][OPERATOR][VALUE] +func (r factory) parse(condition string) (Constraint, error) { + if strings.TrimSpace(condition) == "" { + return nil, nil + } + + operators := []string{ + notEqual, + lessEqual, + greaterEqual, + equal, + less, + greater, + } + + propertyEnd := strings.IndexAny(condition, "<>=!") + if propertyEnd == -1 { + return nil, fmt.Errorf("invalid constraint: %v", condition) + } + + property := condition[:propertyEnd] + condition = strings.TrimPrefix(condition, property) + + p, ok := r.properties[property] + if !ok || p == nil { + return nil, nil + } + + var op string + for _, o := range operators { + if strings.HasPrefix(condition, o) { + op = o + break + } + } + value := strings.TrimPrefix(condition, op) + + c := binary{ + left: p, + right: value, + operator: op, + } + return c, p.Validate(value) +} diff --git a/internal/requirements/constraints/factory_test.go b/internal/requirements/constraints/factory_test.go new file mode 100644 index 00000000..019209f0 --- /dev/null +++ b/internal/requirements/constraints/factory_test.go @@ -0,0 +1,187 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints + +import ( + "testing" + + testlog "github.com/sirupsen/logrus/hooks/test" + "github.com/stretchr/testify/require" +) + +func TestParse(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + cuda := NewVersionProperty("cuda", "") + + f := factory{ + logger: logger, + properties: map[string]Property{ + "cuda": cuda, + }, + } + + testCases := []struct { + description string + condition string + expectedError bool + expected Constraint + }{ + { + description: "empty is nil", + condition: "", + expected: nil, + }, + { + description: "missing operator is invalid", + condition: "notvalid", + expectedError: true, + }, + { + description: "invalid property is invalid", + condition: "foo=45", + expected: nil, + }, + { + description: "cuda must be semver", + condition: "cuda=foo", + expectedError: true, + expected: binary{cuda, equal, "foo"}, + }, + { + description: "cuda greater than equal", + condition: "cuda>=11.6", + expected: binary{cuda, greaterEqual, "11.6"}, + }, + { + description: "cuda greater than", + condition: "cuda>11.6", + expected: binary{cuda, greater, "11.6"}, + }, + { + description: "cuda less than equal", + condition: "cuda<=11.6", + expected: binary{cuda, lessEqual, "11.6"}, + }, + { + description: "cuda less than", + condition: "cuda<11.6", + expected: binary{cuda, less, "11.6"}, + }, + { + description: "cuda equal", + condition: "cuda=11.6", + expected: binary{cuda, equal, "11.6"}, + }, + { + description: "cuda not equal", + condition: "cuda!=11.6", + expected: binary{cuda, notEqual, "11.6"}, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + c, err := f.parse(tc.condition) + if tc.expectedError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.EqualValues(t, tc.expected, c) + }) + } +} + +func TestNewConstraintFromRequirement(t *testing.T) { + logger, _ := testlog.NewNullLogger() + + cuda := &PropertyMock{} + arch := &PropertyMock{} + + f := factory{ + logger: logger, + properties: map[string]Property{ + "cuda": cuda, + "arch": arch, + }, + } + + testCases := []struct { + description string + requirement string + expectedError bool + expected Constraint + }{ + { + description: "empty is nil", + requirement: "", + expected: nil, + }, + { + description: "malformed constraint is invalid", + requirement: "notvalid", + expectedError: true, + expected: nil, + }, + { + description: "unsupported property is ignored", + requirement: "cuda>=11.6 foo=bar", + expected: binary{cuda, greaterEqual, "11.6"}, + }, + { + description: "space-separated is and", + requirement: "cuda>=11.6 arch=5.3", + expected: and([]Constraint{ + binary{cuda, greaterEqual, "11.6"}, + binary{arch, equal, "5.3"}, + }), + }, + { + description: "comma-separated is or", + requirement: "cuda>=11.6,arch=5.3", + expected: or([]Constraint{ + binary{cuda, greaterEqual, "11.6"}, + binary{arch, equal, "5.3"}, + }), + }, + { + description: "and takes precedence", + requirement: "cuda<13.6 cuda>=11.6,arch=5.3", + expected: or([]Constraint{ + binary{cuda, less, "13.6"}, + and([]Constraint{ + binary{cuda, greaterEqual, "11.6"}, + binary{arch, equal, "5.3"}, + }), + }), + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + c, err := f.newConstraintFromRequirement(tc.requirement) + if tc.expectedError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + require.EqualValues(t, tc.expected, c) + }) + } + +} diff --git a/internal/requirements/constraints/logical.go b/internal/requirements/constraints/logical.go new file mode 100644 index 00000000..c1dda521 --- /dev/null +++ b/internal/requirements/constraints/logical.go @@ -0,0 +1,91 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints + +import ( + "fmt" + "strings" +) + +// or represents an OR operation on a set of constraints +type or []Constraint + +// and represents an AND (ALL) operation on a set of contraints +type and []Constraint + +// AND constructs a new constraint that is the logical AND of the supplied constraints +func AND(constraints []Constraint) Constraint { + if len(constraints) == 0 { + return &always{} + } + if len(constraints) == 1 { + return constraints[0] + } + return and(constraints) +} + +// OR constructs a new constrant that is the logical OR of the supplied constraints +func OR(constraints []Constraint) Constraint { + if len(constraints) == 0 { + return nil + } + if len(constraints) == 1 { + return constraints[0] + } + + return or(constraints) +} + +func (operands or) Assert() error { + for _, o := range operands { + // We stop on the first nil + if err := o.Assert(); err == nil { + return nil + } + } + return fmt.Errorf("%v not met", operands) +} + +func (operands or) String() string { + var terms []string + + for _, o := range operands { + terms = append(terms, o.String()) + } + + return strings.Join(terms, "||") +} + +func (operands and) Assert() error { + for _, o := range operands { + // We stop on the first Assert + if err := o.Assert(); err != nil { + return err + } + } + return nil +} + +func (operands and) String() string { + var terms []string + + for _, o := range operands { + terms = append(terms, o.String()) + } + + return strings.Join(terms, "&&") +} diff --git a/internal/requirements/constraints/logical_test.go b/internal/requirements/constraints/logical_test.go new file mode 100644 index 00000000..657cf6d8 --- /dev/null +++ b/internal/requirements/constraints/logical_test.go @@ -0,0 +1,152 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestANDConstraint(t *testing.T) { + + never := ConstraintMock{AssertFunc: func() error { return fmt.Errorf("false") }} + + testCases := []struct { + description string + constraints []Constraint + expected bool + }{ + { + description: "empty is always true", + constraints: []Constraint{}, + expected: true, + }, + { + description: "single true constraint is true", + constraints: []Constraint{ + &always{}, + }, + expected: true, + }, + { + description: "single false constraint is false", + constraints: []Constraint{ + &never, + }, + expected: false, + }, + { + description: "multiple true constraints are true", + constraints: []Constraint{ + &always{}, &always{}, + }, + expected: true, + }, + { + description: "mixed constraints are false (first is true)", + constraints: []Constraint{ + &always{}, &never, + }, + expected: false, + }, + { + description: "mixed constraints are false (last is true)", + constraints: []Constraint{ + &never, &always{}, + }, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + err := and(tc.constraints).Assert() + if tc.expected { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } + +} + +func TestORConstraint(t *testing.T) { + + never := ConstraintMock{AssertFunc: func() error { return fmt.Errorf("false") }} + + testCases := []struct { + description string + constraints []Constraint + expected bool + }{ + { + description: "empty is always false", + constraints: []Constraint{}, + expected: false, + }, + { + description: "single true constraint is true", + constraints: []Constraint{ + &always{}, + }, + expected: true, + }, + { + description: "single false constraint is false", + constraints: []Constraint{ + &never, + }, + expected: false, + }, + { + description: "multiple true constraints are true", + constraints: []Constraint{ + &always{}, &always{}, + }, + expected: true, + }, + { + description: "mixed constraints are true (first is true)", + constraints: []Constraint{ + &always{}, &never, + }, + expected: true, + }, + { + description: "mixed constraints are true (last is true)", + constraints: []Constraint{ + &never, &always{}, + }, + expected: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.description, func(t *testing.T) { + err := or(tc.constraints).Assert() + if tc.expected { + require.NoError(t, err) + } else { + require.Error(t, err) + } + }) + } + +} diff --git a/internal/requirements/constraints/property.go b/internal/requirements/constraints/property.go new file mode 100644 index 00000000..852b2b40 --- /dev/null +++ b/internal/requirements/constraints/property.go @@ -0,0 +1,129 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package constraints + +import ( + "fmt" + "strings" + + "golang.org/x/mod/semver" +) + +//go:generate moq -stub -out property_mock.go . Property +// Property represents a property that is used to check requirements +type Property interface { + Name() string + Value() (string, error) + String() string + CompareTo(string) (int, error) + Validate(string) error +} + +// NewStringProperty creates a string property based on the name-value pair +func NewStringProperty(name string, value string) Property { + p := stringProperty{ + name: name, + value: value, + } + + return p +} + +// NewVersionProperty creates a property representing a semantic version based on the name-value pair +func NewVersionProperty(name string, value string) Property { + p := versionProperty{ + stringProperty: stringProperty{ + name: name, + value: value, + }, + } + + return p +} + +// stringProperty represents a property that is used to check requirements +type stringProperty struct { + name string + value string +} + +type versionProperty struct { + stringProperty +} + +// Name returns a stringProperty's name +func (p stringProperty) Name() string { + return p.name +} + +// Value returns a stringProperty's value or an error if this cannot be determined +func (p stringProperty) Value() (string, error) { + return p.value, nil +} + +// CompareTo compares two strings to each other +func (p stringProperty) CompareTo(other string) (int, error) { + value := p.value + + if value < other { + return -1, nil + } + + if value > other { + return 1, nil + } + + return 0, nil +} + +// Validate returns nil for all input strings +func (p stringProperty) Validate(string) error { + return nil +} + +// String returns the string representation of the name value combination +func (p stringProperty) String() string { + v, err := p.Value() + if err != nil { + return fmt.Sprintf("invalid %v: %v", p.name, err) + } + return fmt.Sprintf("%v=%v", p.name, v) +} + +// CompareTo compares two versions to each other as semantic versions +func (p versionProperty) CompareTo(other string) (int, error) { + if err := p.Validate(other); err != nil { + return 0, fmt.Errorf("invailid value for %v: %v", p.name, err) + } + + vValue := ensurePrefix(p.value, "v") + vOther := ensurePrefix(other, "v") + return semver.Compare(vValue, vOther), nil +} + +// Validate checks whether the supplied value is a valid semantic version +func (p versionProperty) Validate(value string) error { + if !semver.IsValid(ensurePrefix(value, "v")) { + return fmt.Errorf("invailid value %v; expected a valid version string", value) + } + + return nil +} + +func ensurePrefix(s string, prefix string) string { + return prefix + strings.TrimPrefix(s, prefix) +} diff --git a/internal/requirements/constraints/property_mock.go b/internal/requirements/constraints/property_mock.go new file mode 100644 index 00000000..656f074a --- /dev/null +++ b/internal/requirements/constraints/property_mock.go @@ -0,0 +1,241 @@ +// Code generated by moq; DO NOT EDIT. +// github.com/matryer/moq + +package constraints + +import ( + "sync" +) + +// Ensure, that PropertyMock does implement Property. +// If this is not the case, regenerate this file with moq. +var _ Property = &PropertyMock{} + +// PropertyMock is a mock implementation of Property. +// +// func TestSomethingThatUsesProperty(t *testing.T) { +// +// // make and configure a mocked Property +// mockedProperty := &PropertyMock{ +// CompareToFunc: func(s string) (int, error) { +// panic("mock out the CompareTo method") +// }, +// NameFunc: func() string { +// panic("mock out the Name method") +// }, +// StringFunc: func() string { +// panic("mock out the String method") +// }, +// ValidateFunc: func(s string) error { +// panic("mock out the Validate method") +// }, +// ValueFunc: func() (string, error) { +// panic("mock out the Value method") +// }, +// } +// +// // use mockedProperty in code that requires Property +// // and then make assertions. +// +// } +type PropertyMock struct { + // CompareToFunc mocks the CompareTo method. + CompareToFunc func(s string) (int, error) + + // NameFunc mocks the Name method. + NameFunc func() string + + // StringFunc mocks the String method. + StringFunc func() string + + // ValidateFunc mocks the Validate method. + ValidateFunc func(s string) error + + // ValueFunc mocks the Value method. + ValueFunc func() (string, error) + + // calls tracks calls to the methods. + calls struct { + // CompareTo holds details about calls to the CompareTo method. + CompareTo []struct { + // S is the s argument value. + S string + } + // Name holds details about calls to the Name method. + Name []struct { + } + // String holds details about calls to the String method. + String []struct { + } + // Validate holds details about calls to the Validate method. + Validate []struct { + // S is the s argument value. + S string + } + // Value holds details about calls to the Value method. + Value []struct { + } + } + lockCompareTo sync.RWMutex + lockName sync.RWMutex + lockString sync.RWMutex + lockValidate sync.RWMutex + lockValue sync.RWMutex +} + +// CompareTo calls CompareToFunc. +func (mock *PropertyMock) CompareTo(s string) (int, error) { + callInfo := struct { + S string + }{ + S: s, + } + mock.lockCompareTo.Lock() + mock.calls.CompareTo = append(mock.calls.CompareTo, callInfo) + mock.lockCompareTo.Unlock() + if mock.CompareToFunc == nil { + var ( + nOut int + errOut error + ) + return nOut, errOut + } + return mock.CompareToFunc(s) +} + +// CompareToCalls gets all the calls that were made to CompareTo. +// Check the length with: +// len(mockedProperty.CompareToCalls()) +func (mock *PropertyMock) CompareToCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockCompareTo.RLock() + calls = mock.calls.CompareTo + mock.lockCompareTo.RUnlock() + return calls +} + +// Name calls NameFunc. +func (mock *PropertyMock) Name() string { + callInfo := struct { + }{} + mock.lockName.Lock() + mock.calls.Name = append(mock.calls.Name, callInfo) + mock.lockName.Unlock() + if mock.NameFunc == nil { + var ( + sOut string + ) + return sOut + } + return mock.NameFunc() +} + +// NameCalls gets all the calls that were made to Name. +// Check the length with: +// len(mockedProperty.NameCalls()) +func (mock *PropertyMock) NameCalls() []struct { +} { + var calls []struct { + } + mock.lockName.RLock() + calls = mock.calls.Name + mock.lockName.RUnlock() + return calls +} + +// String calls StringFunc. +func (mock *PropertyMock) String() string { + callInfo := struct { + }{} + mock.lockString.Lock() + mock.calls.String = append(mock.calls.String, callInfo) + mock.lockString.Unlock() + if mock.StringFunc == nil { + var ( + sOut string + ) + return sOut + } + return mock.StringFunc() +} + +// StringCalls gets all the calls that were made to String. +// Check the length with: +// len(mockedProperty.StringCalls()) +func (mock *PropertyMock) StringCalls() []struct { +} { + var calls []struct { + } + mock.lockString.RLock() + calls = mock.calls.String + mock.lockString.RUnlock() + return calls +} + +// Validate calls ValidateFunc. +func (mock *PropertyMock) Validate(s string) error { + callInfo := struct { + S string + }{ + S: s, + } + mock.lockValidate.Lock() + mock.calls.Validate = append(mock.calls.Validate, callInfo) + mock.lockValidate.Unlock() + if mock.ValidateFunc == nil { + var ( + errOut error + ) + return errOut + } + return mock.ValidateFunc(s) +} + +// ValidateCalls gets all the calls that were made to Validate. +// Check the length with: +// len(mockedProperty.ValidateCalls()) +func (mock *PropertyMock) ValidateCalls() []struct { + S string +} { + var calls []struct { + S string + } + mock.lockValidate.RLock() + calls = mock.calls.Validate + mock.lockValidate.RUnlock() + return calls +} + +// Value calls ValueFunc. +func (mock *PropertyMock) Value() (string, error) { + callInfo := struct { + }{} + mock.lockValue.Lock() + mock.calls.Value = append(mock.calls.Value, callInfo) + mock.lockValue.Unlock() + if mock.ValueFunc == nil { + var ( + sOut string + errOut error + ) + return sOut, errOut + } + return mock.ValueFunc() +} + +// ValueCalls gets all the calls that were made to Value. +// Check the length with: +// len(mockedProperty.ValueCalls()) +func (mock *PropertyMock) ValueCalls() []struct { +} { + var calls []struct { + } + mock.lockValue.RLock() + calls = mock.calls.Value + mock.lockValue.RUnlock() + return calls +} diff --git a/internal/requirements/requirements.go b/internal/requirements/requirements.go new file mode 100644 index 00000000..bf51cc96 --- /dev/null +++ b/internal/requirements/requirements.go @@ -0,0 +1,69 @@ +/** +# Copyright (c) 2022, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +**/ + +package requirements + +import ( + "github.com/NVIDIA/nvidia-container-toolkit/internal/requirements/constraints" + "github.com/sirupsen/logrus" +) + +// Requirements represents a collection of requirements that can be compared to properties +type Requirements struct { + logger *logrus.Logger + requirements []string + properties map[string]constraints.Property +} + +// New creates a new set of requirements +func New(logger *logrus.Logger, requirements []string) *Requirements { + r := Requirements{ + logger: logger, + requirements: requirements, + properties: map[string]constraints.Property{ + // Set up the supported properties. These are overridden with actual values. + CUDA: constraints.NewVersionProperty(CUDA, ""), + ARCH: constraints.NewVersionProperty(ARCH, ""), + DRIVER: constraints.NewVersionProperty(DRIVER, ""), + BRAND: constraints.NewStringProperty(BRAND, ""), + }, + } + + return &r +} + +// AddVersionProperty adds the specified property (name, value pair) to the requirements +func (r *Requirements) AddVersionProperty(name string, value string) { + r.properties[name] = constraints.NewVersionProperty(name, value) +} + +// AddStringProperty adds the specified property (name, value pair) to the requirements +func (r *Requirements) AddStringProperty(name string, value string) { + r.properties[name] = constraints.NewStringProperty(name, value) +} + +// Assert checks the specified requirements +func (r Requirements) Assert() error { + if len(r.requirements) == 0 { + return nil + } + + c, err := constraints.New(r.logger, r.requirements, r.properties) + if err != nil { + return err + } + return c.Assert() +} From 49f4bb3198dc5031ac396bf7b888dda248e71529 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Mon, 2 May 2022 10:28:25 +0200 Subject: [PATCH 10/16] Check requirements before creating CSV discoverer Signed-off-by: Evan Lezar --- .../modifier/experimental.go | 41 ++++++++++++++++++- .../modifier/experimental_test.go | 4 +- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/cmd/nvidia-container-runtime/modifier/experimental.go b/cmd/nvidia-container-runtime/modifier/experimental.go index 2a048369..4993c147 100644 --- a/cmd/nvidia-container-runtime/modifier/experimental.go +++ b/cmd/nvidia-container-runtime/modifier/experimental.go @@ -22,10 +22,13 @@ import ( "strings" "github.com/NVIDIA/nvidia-container-toolkit/internal/config" + "github.com/NVIDIA/nvidia-container-toolkit/internal/config/image" + "github.com/NVIDIA/nvidia-container-toolkit/internal/cuda" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover" "github.com/NVIDIA/nvidia-container-toolkit/internal/discover/csv" "github.com/NVIDIA/nvidia-container-toolkit/internal/edits" "github.com/NVIDIA/nvidia-container-toolkit/internal/oci" + "github.com/NVIDIA/nvidia-container-toolkit/internal/requirements" "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -46,7 +49,7 @@ const ( // NewExperimentalModifier creates a modifier that applies the experimental // modications to an OCI spec if required by the runtime wrapper. func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) { - _, err := ociSpec.Load() + rawSpec, err := ociSpec.Load() if err != nil { return nil, fmt.Errorf("failed to load OCI spec: %v", err) } @@ -75,6 +78,17 @@ func NewExperimentalModifier(logger *logrus.Logger, cfg *config.Config, ociSpec } d = legacyDiscoverer case "csv": + // TODO: Once the devices have been encapsulated in the CUDA image, this can be moved to before the + // visible devices are checked. + image, err := image.NewCUDAImageFromSpec(rawSpec) + if err != nil { + return nil, err + } + + if err := checkRequirements(logger, &image); err != nil { + return nil, fmt.Errorf("requirements not met: %v", err) + } + csvFiles, err := csv.GetFileList(csv.DefaultMountSpecPath) if err != nil { return nil, fmt.Errorf("failed to get list of CSV files: %v", err) @@ -134,6 +148,31 @@ func (m experimental) Modify(spec *specs.Spec) error { return specEdits.Modify(spec) } +func checkRequirements(logger *logrus.Logger, image *image.CUDA) error { + if image.HasDisableRequire() { + // TODO: We could print the real value here instead + logger.Debugf("NVIDIA_DISABLE_REQUIRE=%v; skipping requirement checks", true) + return nil + } + + imageRequirements, err := image.GetRequirements() + if err != nil { + // TODO: Should we treat this as a failure, or just issue a warning? + return fmt.Errorf("failed to get image requirements: %v", err) + } + + r := requirements.New(logger, imageRequirements) + + cudaVersion, err := cuda.Version() + if err != nil { + logger.Warnf("Failed to get CUDA version: %v", err) + } else { + r.AddVersionProperty(requirements.CUDA, cudaVersion) + } + + return r.Assert() +} + // resolveAutoDiscoverMode determines the correct discover mode for the specified platform if set to "auto" func resolveAutoDiscoverMode(logger *logrus.Logger, mode string) (rmode string) { if mode != "auto" { diff --git a/cmd/nvidia-container-runtime/modifier/experimental_test.go b/cmd/nvidia-container-runtime/modifier/experimental_test.go index 34f83bf4..cd492000 100644 --- a/cmd/nvidia-container-runtime/modifier/experimental_test.go +++ b/cmd/nvidia-container-runtime/modifier/experimental_test.go @@ -42,8 +42,8 @@ func TestNewExperimentalModifier(t *testing.T) { { description: "spec load error returns error", spec: &oci.SpecMock{ - LoadFunc: func() error { - return fmt.Errorf("load failed") + LoadFunc: func() (*specs.Spec, error) { + return nil, fmt.Errorf("load failed") }, }, expectedError: fmt.Errorf("load failed"), From 9d2363e12efd7266cc4d38630291fd90757038ec Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 4 May 2022 14:06:28 +0200 Subject: [PATCH 11/16] Return low-level runtime if subcommand is not create This also removes a test that invokes nvidia-container-runtime run --bundle expecting an error. This test is no longer valid since this command line is forwared to runc where the error should be detected. Signed-off-by: Evan Lezar --- cmd/nvidia-container-runtime/main_test.go | 5 ----- .../runtime_factory.go | 18 ++++++++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/cmd/nvidia-container-runtime/main_test.go b/cmd/nvidia-container-runtime/main_test.go index d6ba14ae..ff507f3b 100644 --- a/cmd/nvidia-container-runtime/main_test.go +++ b/cmd/nvidia-container-runtime/main_test.go @@ -80,11 +80,6 @@ func TestBadInput(t *testing.T) { t.Fatal(err) } - cmdRun := exec.Command(nvidiaRuntime, "run", "--bundle") - t.Logf("executing: %s\n", strings.Join(cmdRun.Args, " ")) - output, err := cmdRun.CombinedOutput() - require.Errorf(t, err, "runtime should return an error", "output=%v", string(output)) - cmdCreate := exec.Command(nvidiaRuntime, "create", "--bundle") t.Logf("executing: %s\n", strings.Join(cmdCreate.Args, " ")) err = cmdCreate.Run() diff --git a/cmd/nvidia-container-runtime/runtime_factory.go b/cmd/nvidia-container-runtime/runtime_factory.go index 43d03769..ca06ca17 100644 --- a/cmd/nvidia-container-runtime/runtime_factory.go +++ b/cmd/nvidia-container-runtime/runtime_factory.go @@ -33,10 +33,6 @@ const ( // newNVIDIAContainerRuntime is a factory method that constructs a runtime based on the selected configuration and specified logger func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv []string) (oci.Runtime, error) { - ociSpec, err := oci.NewSpec(logger, argv) - if err != nil { - return nil, fmt.Errorf("error constructing OCI specification: %v", err) - } lowLevelRuntimeCandidates := []string{dockerRuncExecutableName, runcExecutableName} lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, lowLevelRuntimeCandidates) @@ -44,7 +40,17 @@ func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv [ return nil, fmt.Errorf("error constructing low-level runtime: %v", err) } - specModifier, err := newSpecModifier(logger, cfg, ociSpec) + if !oci.HasCreateSubcommand(argv) { + logger.Debugf("Skipping modifier for non-create subcommand") + return lowLevelRuntime, nil + } + + ociSpec, err := oci.NewSpec(logger, argv) + if err != nil { + return nil, fmt.Errorf("error constructing OCI specification: %v", err) + } + + specModifier, err := newSpecModifier(logger, cfg, ociSpec, argv) if err != nil { return nil, fmt.Errorf("failed to construct OCI spec modifier: %v", err) } @@ -61,7 +67,7 @@ func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv [ } // newSpecModifier is a factory method that creates constructs an OCI spec modifer based on the provided config. -func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec) (oci.SpecModifier, error) { +func newSpecModifier(logger *logrus.Logger, cfg *config.Config, ociSpec oci.Spec, argv []string) (oci.SpecModifier, error) { if !cfg.NVIDIAContainerRuntimeConfig.Experimental { return modifier.NewStableRuntimeModifier(logger), nil } From 75ce057878c8bf5155f0cd79b0b153f97ef1f8ad Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 5 May 2022 13:45:27 +0200 Subject: [PATCH 12/16] Add debug log for command line arguments Signed-off-by: Evan Lezar --- cmd/nvidia-container-runtime/main.go | 2 +- cmd/nvidia-container-runtime/runtime_factory.go | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/nvidia-container-runtime/main.go b/cmd/nvidia-container-runtime/main.go index 099c47be..be9054d7 100644 --- a/cmd/nvidia-container-runtime/main.go +++ b/cmd/nvidia-container-runtime/main.go @@ -21,7 +21,6 @@ func main() { // run is an entry point that allows for idiomatic handling of errors // when calling from the main function. func run(argv []string) (rerr error) { - logger.Debugf("Running %v", argv) cfg, err := config.GetConfig() if err != nil { return fmt.Errorf("error loading config: %v", err) @@ -45,6 +44,7 @@ func run(argv []string) (rerr error) { logger.Warnf("Invalid log-level '%v'; using '%v'", cfg.NVIDIAContainerRuntimeConfig.LogLevel, logger.Level.String()) } + logger.Debugf("Command line arguments: %v", argv) runtime, err := newNVIDIAContainerRuntime(logger.Logger, cfg, argv) if err != nil { return fmt.Errorf("failed to create NVIDIA Container Runtime: %v", err) diff --git a/cmd/nvidia-container-runtime/runtime_factory.go b/cmd/nvidia-container-runtime/runtime_factory.go index ca06ca17..a9d7a7ab 100644 --- a/cmd/nvidia-container-runtime/runtime_factory.go +++ b/cmd/nvidia-container-runtime/runtime_factory.go @@ -33,7 +33,6 @@ const ( // newNVIDIAContainerRuntime is a factory method that constructs a runtime based on the selected configuration and specified logger func newNVIDIAContainerRuntime(logger *logrus.Logger, cfg *config.Config, argv []string) (oci.Runtime, error) { - lowLevelRuntimeCandidates := []string{dockerRuncExecutableName, runcExecutableName} lowLevelRuntime, err := oci.NewLowLevelRuntime(logger, lowLevelRuntimeCandidates) if err != nil { From 9f50ac95c42f44807838b924ce56831413fc7e5a Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 5 May 2022 14:09:28 +0200 Subject: [PATCH 13/16] Add CUDA ComputeCapability function Signed-off-by: Evan Lezar --- internal/cuda/cuda.go | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/internal/cuda/cuda.go b/internal/cuda/cuda.go index 6224de3e..2c70a821 100644 --- a/internal/cuda/cuda.go +++ b/internal/cuda/cuda.go @@ -31,11 +31,21 @@ import ( #define CUDAAPI #endif +typedef int CUdevice; + +typedef enum CUdevice_attribute_enum { + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR = 75, + CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR = 76 +} CUdevice_attribute; + typedef enum cudaError_enum { CUDA_SUCCESS = 0 } CUresult; +CUresult CUDAAPI cuInit(unsigned int Flags); CUresult CUDAAPI cuDriverGetVersion(int *driverVersion); +CUresult CUDAAPI cuDeviceGet(CUdevice *device, int ordinal); +CUresult CUDAAPI cuDeviceGetAttribute(int *pi, CUdevice_attribute attrib, CUdevice dev); */ import "C" @@ -71,6 +81,48 @@ func Version() (string, error) { return fmt.Sprintf("%d.%d", major, minor), nil } +// ComputeCapability returns the CUDA compute capability of a device with the specified index as a string +// or an error if this cannot be determined. +func ComputeCapability(index int) (string, error) { + lib, err := load() + if err != nil { + return "", err + } + defer lib.Close() + + if err := lib.Lookup("cuInit"); err != nil { + return "", fmt.Errorf("failed to lookup symbol: %v", err) + } + if err := lib.Lookup("cuDeviceGet"); err != nil { + return "", fmt.Errorf("failed to lookup symbol: %v", err) + } + if err := lib.Lookup("cuDeviceGetAttribute"); err != nil { + return "", fmt.Errorf("failed to lookup symbol: %v", err) + } + + if result := C.cuInit(C.uint(0)); result != C.CUDA_SUCCESS { + return "", fmt.Errorf("failed to initialize CUDA: result=%v", result) + } + + var device C.CUdevice + // NOTE: We only query the first device + if result := C.cuDeviceGet(&device, C.int(index)); result != C.CUDA_SUCCESS { + return "", fmt.Errorf("failed to get CUDA device %v: result=%v", 0, result) + } + + var major C.int + if result := C.cuDeviceGetAttribute(&major, C.CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MAJOR, device); result != C.CUDA_SUCCESS { + return "", fmt.Errorf("failed to get CUDA compute capability major for device %v : result=%v", 0, result) + } + + var minor C.int + if result := C.cuDeviceGetAttribute(&minor, C.CU_DEVICE_ATTRIBUTE_COMPUTE_CAPABILITY_MINOR, device); result != C.CUDA_SUCCESS { + return "", fmt.Errorf("failed to get CUDA compute capability minor for device %v: result=%v", 0, result) + } + + return fmt.Sprintf("%d.%d", major, minor), nil +} + func load() (*dl.DynamicLibrary, error) { lib := dl.New(libraryName, libraryLoadFlags) if lib == nil { From 70c45881972155f3be83737711321c877f995ee3 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 5 May 2022 14:11:30 +0200 Subject: [PATCH 14/16] Add compute capability of first device as arch property Signed-off-by: Evan Lezar --- cmd/nvidia-container-runtime/modifier/experimental.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/nvidia-container-runtime/modifier/experimental.go b/cmd/nvidia-container-runtime/modifier/experimental.go index 4993c147..0bf73bd8 100644 --- a/cmd/nvidia-container-runtime/modifier/experimental.go +++ b/cmd/nvidia-container-runtime/modifier/experimental.go @@ -170,6 +170,13 @@ func checkRequirements(logger *logrus.Logger, image *image.CUDA) error { r.AddVersionProperty(requirements.CUDA, cudaVersion) } + compteCapability, err := cuda.ComputeCapability(0) + if err != nil { + logger.Warnf("Failed to get CUDA Compute Capability: %v", err) + } else { + r.AddVersionProperty(requirements.ARCH, compteCapability) + } + return r.Assert() } From 9e46d41dbedb5a614f7abab8eae707e261fad9e6 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Thu, 5 May 2022 14:14:01 +0200 Subject: [PATCH 15/16] Add debug logging when checking requirements Signed-off-by: Evan Lezar --- internal/requirements/requirements.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/requirements/requirements.go b/internal/requirements/requirements.go index bf51cc96..5305ce28 100644 --- a/internal/requirements/requirements.go +++ b/internal/requirements/requirements.go @@ -61,6 +61,7 @@ func (r Requirements) Assert() error { return nil } + r.logger.Debugf("Checking properties %+v against requirements %v", r.properties, r.requirements) c, err := constraints.New(r.logger, r.requirements, r.properties) if err != nil { return err From 785f120c312d3de204808b580817d356a19b21d8 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Fri, 6 May 2022 13:22:34 +0200 Subject: [PATCH 16/16] Fix form -> from in comment Signed-off-by: Evan Lezar --- internal/config/image/cuda_image.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/config/image/cuda_image.go b/internal/config/image/cuda_image.go index 2e986773..c92b75a5 100644 --- a/internal/config/image/cuda_image.go +++ b/internal/config/image/cuda_image.go @@ -72,7 +72,7 @@ func (i CUDA) IsLegacy() bool { return len(legacyCudaVersion) > 0 && len(cudaRequire) == 0 } -// GetRequirements returns the requirements fomr all NVIDIA_REQUIRE_ environment +// GetRequirements returns the requirements from all NVIDIA_REQUIRE_ environment // variables. func (i CUDA) GetRequirements() ([]string, error) { // TODO: We need not process this if disable require is set, but this will be done