Merge branch 'deduplicate-symlinks' into 'main'

Deduplicate symlinks

See merge request nvidia/container-toolkit/container-toolkit!493

internal/ldcache/empty.go

@@ -0,0 +1,37 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package ldcache
+
+import "github.com/NVIDIA/nvidia-container-toolkit/internal/logger"
+
+type empty struct {
+	logger logger.Interface
+	path   string
+}
+
+var _ LDCache = (*empty)(nil)
+
+// List always returns nil for an empty ldcache
+func (e *empty) List() ([]string, []string) {
+	return nil, nil
+}
+
+// Lookup logs a debug message and returns nil for an empty ldcache
+func (e *empty) Lookup(prefixes ...string) ([]string, []string) {
+	e.logger.Debugf("Calling Lookup(%v) on empty ldcache: %v", prefixes, e.path)
+	return nil, nil
+}

internal/ldcache/ldcache.go

@@ -81,6 +81,8 @@ type entry2 struct {
 }
 
 // LDCache represents the interface for performing lookups into the LDCache
+//
+//go:generate moq -out ldcache_mock.go . LDCache
 type LDCache interface {
 	List() ([]string, []string)
 	Lookup(...string) ([]string, []string)
@@ -103,7 +105,14 @@ func New(logger logger.Interface, root string) (LDCache, error) {
 
 	logger.Debugf("Opening ld.conf at %v", path)
 	f, err := os.Open(path)
-	if err != nil {
+	if os.IsNotExist(err) {
+		logger.Warningf("Could not find ld.so.cache at %v; creating empty cache", path)
+		e := &empty{
+			logger: logger,
+			path:   path,
+		}
+		return e, nil
+	} else if err != nil {
 		return nil, err
 	}
 	defer f.Close()

internal/ldcache/ldcache_mock.go

@@ -0,0 +1,111 @@
+// Code generated by moq; DO NOT EDIT.
+// github.com/matryer/moq
+
+package ldcache
+
+import (
+	"sync"
+)
+
+// Ensure, that LDCacheMock does implement LDCache.
+// If this is not the case, regenerate this file with moq.
+var _ LDCache = &LDCacheMock{}
+
+// LDCacheMock is a mock implementation of LDCache.
+//
+//	func TestSomethingThatUsesLDCache(t *testing.T) {
+//
+//		// make and configure a mocked LDCache
+//		mockedLDCache := &LDCacheMock{
+//			ListFunc: func() ([]string, []string) {
+//				panic("mock out the List method")
+//			},
+//			LookupFunc: func(strings ...string) ([]string, []string) {
+//				panic("mock out the Lookup method")
+//			},
+//		}
+//
+//		// use mockedLDCache in code that requires LDCache
+//		// and then make assertions.
+//
+//	}
+type LDCacheMock struct {
+	// ListFunc mocks the List method.
+	ListFunc func() ([]string, []string)
+
+	// LookupFunc mocks the Lookup method.
+	LookupFunc func(strings ...string) ([]string, []string)
+
+	// calls tracks calls to the methods.
+	calls struct {
+		// List holds details about calls to the List method.
+		List []struct {
+		}
+		// Lookup holds details about calls to the Lookup method.
+		Lookup []struct {
+			// Strings is the strings argument value.
+			Strings []string
+		}
+	}
+	lockList   sync.RWMutex
+	lockLookup sync.RWMutex
+}
+
+// List calls ListFunc.
+func (mock *LDCacheMock) List() ([]string, []string) {
+	if mock.ListFunc == nil {
+		panic("LDCacheMock.ListFunc: method is nil but LDCache.List was just called")
+	}
+	callInfo := struct {
+	}{}
+	mock.lockList.Lock()
+	mock.calls.List = append(mock.calls.List, callInfo)
+	mock.lockList.Unlock()
+	return mock.ListFunc()
+}
+
+// ListCalls gets all the calls that were made to List.
+// Check the length with:
+//
+//	len(mockedLDCache.ListCalls())
+func (mock *LDCacheMock) ListCalls() []struct {
+} {
+	var calls []struct {
+	}
+	mock.lockList.RLock()
+	calls = mock.calls.List
+	mock.lockList.RUnlock()
+	return calls
+}
+
+// Lookup calls LookupFunc.
+func (mock *LDCacheMock) Lookup(strings ...string) ([]string, []string) {
+	if mock.LookupFunc == nil {
+		panic("LDCacheMock.LookupFunc: method is nil but LDCache.Lookup was just called")
+	}
+	callInfo := struct {
+		Strings []string
+	}{
+		Strings: strings,
+	}
+	mock.lockLookup.Lock()
+	mock.calls.Lookup = append(mock.calls.Lookup, callInfo)
+	mock.lockLookup.Unlock()
+	return mock.LookupFunc(strings...)
+}
+
+// LookupCalls gets all the calls that were made to Lookup.
+// Check the length with:
+//
+//	len(mockedLDCache.LookupCalls())
+func (mock *LDCacheMock) LookupCalls() []struct {
+	Strings []string
+} {
+	var calls []struct {
+		Strings []string
+	}
+	mock.lockLookup.RLock()
+	calls = mock.calls.Lookup
+	mock.lockLookup.RUnlock()
+	return calls
+}

internal/lookup/library.go

@@ -66,7 +66,7 @@ func newLdcacheLocator(logger logger.Interface, root string) Locator {
 		return nil
 	}
 
-	return ldcacheLocator{
+	return &ldcacheLocator{
 		logger: logger,
 		cache:  cache,
 	}
@@ -82,7 +82,7 @@ func (l ldcacheLocator) Locate(libname string) ([]string, error) {
 	}
 
 	if len(paths64) == 0 {
-		return nil, fmt.Errorf("64-bit library %v not found", libname)
+		return nil, fmt.Errorf("64-bit library %v: %w", libname, errNotFound)
 	}
 
 	return paths64, nil

internal/lookup/library_test.go

@@ -0,0 +1,190 @@
+/**
+# Copyright (c) NVIDIA CORPORATION.  All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+**/
+
+package lookup
+
+import (
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+
+	"github.com/NVIDIA/nvidia-container-toolkit/internal/ldcache"
+	testlog "github.com/sirupsen/logrus/hooks/test"
+	"github.com/stretchr/testify/require"
+)
+
+func TestLDCacheLocator(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testDir := t.TempDir()
+	symlinkDir := filepath.Join(testDir, "/lib/symlink")
+	require.NoError(t, os.MkdirAll(symlinkDir, 0755))
+
+	versionLib := filepath.Join(symlinkDir, "libcuda.so.1.2.3")
+	soLink := filepath.Join(symlinkDir, "libcuda.so")
+	sonameLink := filepath.Join(symlinkDir, "libcuda.so.1")
+
+	_, err := os.Create(versionLib)
+	require.NoError(t, err)
+	require.NoError(t, os.Symlink(versionLib, sonameLink))
+	require.NoError(t, os.Symlink(sonameLink, soLink))
+
+	lut := newLdcacheLocator(logger, testDir)
+
+	testCases := []struct {
+		description   string
+		libname       string
+		ldcacheMap    map[string]string
+		expected      []string
+		expectedError error
+	}{
+		{
+			description: "lib only resolves in LDCache",
+			libname:     "libcuda.so",
+			ldcacheMap: map[string]string{
+				"libcuda.so": "/lib/from/ldcache/libcuda.so.4.5.6",
+			},
+			expected: []string{"/lib/from/ldcache/libcuda.so.4.5.6"},
+		},
+		{
+			description:   "lib only not in LDCache returns error",
+			libname:       "libnotcuda.so",
+			expectedError: errNotFound,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			// We override the LDCache with a mock implementation
+			l := lut.(*ldcacheLocator)
+			l.cache = &ldcache.LDCacheMock{
+				LookupFunc: func(strings ...string) ([]string, []string) {
+					var result []string
+					for _, s := range strings {
+						if v, ok := tc.ldcacheMap[s]; ok {
+							result = append(result, v)
+						}
+					}
+					return nil, result
+				},
+			}
+
+			candidates, err := lut.Locate(tc.libname)
+			require.ErrorIs(t, err, tc.expectedError)
+
+			var cleanedCandidates []string
+			for _, c := range candidates {
+				// On MacOS /var and /tmp symlink to /private/var and /private/tmp which is included in the resolved path.
+				cleanedCandidates = append(cleanedCandidates, strings.TrimPrefix(c, "/private"))
+			}
+			require.EqualValues(t, tc.expected, cleanedCandidates)
+		})
+	}
+
+}
+
+func TestLibraryLocator(t *testing.T) {
+	logger, _ := testlog.NewNullLogger()
+
+	testDir := t.TempDir()
+	symlinkDir := filepath.Join(testDir, "/lib/symlink")
+	require.NoError(t, os.MkdirAll(symlinkDir, 0755))
+
+	versionLib := filepath.Join(symlinkDir, "libcuda.so.1.2.3")
+	soLink := filepath.Join(symlinkDir, "libcuda.so")
+	sonameLink := filepath.Join(symlinkDir, "libcuda.so.1")
+
+	f, err := os.Create(versionLib)
+	require.NoError(t, err)
+	f.Close()
+	require.NoError(t, os.Symlink(versionLib, sonameLink))
+	require.NoError(t, os.Symlink(sonameLink, soLink))
+
+	// We create a set of symlinks for duplicate resolution
+	libTarget1 := filepath.Join(symlinkDir, "libtarget.so.1.2.3")
+	source1 := filepath.Join(symlinkDir, "libsource1.so")
+	source2 := filepath.Join(symlinkDir, "libsource2.so")
+
+	target1, err := os.Create(libTarget1)
+	require.NoError(t, err)
+	target1.Close()
+	require.NoError(t, os.Symlink(libTarget1, source1))
+	require.NoError(t, os.Symlink(source1, source2))
+
+	lut, err := NewLibraryLocator(logger, testDir)
+	require.NoError(t, err)
+
+	testCases := []struct {
+		description   string
+		libname       string
+		expected      []string
+		expectedError error
+	}{
+		{
+			description: "slash in path resoves symlink",
+			libname:     "/lib/symlink/libcuda.so",
+			expected:    []string{filepath.Join(testDir, "/lib/symlink/libcuda.so.1.2.3")},
+		},
+		{
+			description: "slash in path resoves symlink",
+			libname:     "/lib/symlink/libcuda.so.1",
+			expected:    []string{filepath.Join(testDir, "/lib/symlink/libcuda.so.1.2.3")},
+		},
+		{
+			description: "slash in path with pattern resolves symlinks",
+			libname:     "/lib/symlink/libcuda.so.*",
+			expected:    []string{filepath.Join(testDir, "/lib/symlink/libcuda.so.1.2.3")},
+		},
+		{
+			description:   "library not found returns error",
+			libname:       "/lib/symlink/libnotcuda.so",
+			expectedError: errNotFound,
+		},
+		{
+			description: "slash in path with pattern resoves symlink",
+			libname:     "/lib/symlink/libcuda.so.*.*.*",
+			expected:    []string{filepath.Join(testDir, "/lib/symlink/libcuda.so.1.2.3")},
+		},
+		{
+			description: "symlinks are deduplicated",
+			libname:     "/lib/symlink/libsource*.so",
+			expected:    []string{filepath.Join(testDir, "/lib/symlink/libtarget.so.1.2.3")},
+		},
+		{
+			description: "pattern resolves to multiple targets",
+			libname:     "/lib/symlink/lib*.so.1.2.3",
+			expected: []string{
+				filepath.Join(testDir, "/lib/symlink/libcuda.so.1.2.3"),
+				filepath.Join(testDir, "/lib/symlink/libtarget.so.1.2.3"),
+			},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			candidates, err := lut.Locate(tc.libname)
+			require.ErrorIs(t, err, tc.expectedError)
+
+			var cleanedCandidates []string
+			for _, c := range candidates {
+				// On MacOS /var and /tmp symlink to /private/var and /private/tmp which is included in the resolved path.
+				cleanedCandidates = append(cleanedCandidates, strings.TrimPrefix(c, "/private"))
+			}
+			require.EqualValues(t, tc.expected, cleanedCandidates)
+		})
+	}
+}

internal/lookup/locator.go

@@ -16,9 +16,13 @@
 
 package lookup
 
+import "errors"
+
 //go:generate moq -stub -out locator_mock.go . Locator
 
 // Locator defines the interface for locating files on a system.
 type Locator interface {
 	Locate(string) ([]string, error)
 }
+
+var errNotFound = errors.New("not found")

internal/lookup/symlinks.go

@@ -103,14 +103,19 @@ func (p symlink) Locate(pattern string) ([]string, error) {
 	if err != nil {
 		return nil, err
 	}
-	if len(candidates) != 1 {
-		return nil, fmt.Errorf("failed to uniquely resolve symlink %v: %v", pattern, candidates)
-	}
 
-	target, err := filepath.EvalSymlinks(candidates[0])
-	if err != nil {
-		return nil, fmt.Errorf("failed to resolve link: %v", err)
+	var targets []string
+	seen := make(map[string]bool)
+	for _, candidate := range candidates {
+		target, err := filepath.EvalSymlinks(candidate)
+		if err != nil {
+			return nil, fmt.Errorf("failed to resolve link: %w", err)
+		}
+		if seen[target] {
+			continue
+		}
+		seen[target] = true
+		targets = append(targets, target)
 	}
-
-	return []string{target}, err
+	return targets, err
 }