Skip to content

Find, verify, and analyze leaked credentials

Confine symlink state handling to scanSymlink in Filesystem source (#4807)

Charlie Gunyonafd5336caad0
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index 0760fccee..5aced6666 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -125,26 +125,15 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _
}
if fileInfo.Mode()&os.ModeSymlink != 0 {
- if !s.canFollowSymlinks() {
- // If the file or directory is a symlink but the followSymlinks is disable ignore the path
- logger.Info("skipping, following symlinks is not allowed", "path", cleanPath)
- continue
- }
// if the root path is a symlink we scan the symlink
ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(s.concurrency)
- initialDepth := 1
- err = s.scanSymlink(ctx, chunksChan, workerPool, rootPath, initialDepth, cleanPath)
- _ = workerPool.Wait()
+ initialDepth := 0
+ err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else if fileInfo.IsDir() {
ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(s.concurrency)
- initialDepth := 1
- err = s.scanDir(ctx, chunksChan, workerPool, rootPath, initialDepth, cleanPath)
- _ = workerPool.Wait()
+ initialDepth := 0
+ err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else {
if !fileInfo.Mode().IsRegular() {
@@ -156,9 +145,7 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _
}
if err != nil && !errors.Is(err, io.EOF) {
- if !errors.Is(err, skipSymlinkErr) {
- logger.Error(err, "error scanning filesystem")
- }
+ logger.Error(err, "error scanning filesystem")
}
}
@@ -168,14 +155,22 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _
func (s *Source) scanSymlink(
ctx trContext.Context,
chunksChan chan *sources.Chunk,
- workerPool *errgroup.Group,
rootPath string,
depth int,
path string,
) error {
+ if !s.canFollowSymlinks() {
+ // If the file or directory is a symlink but the followSymlinks is disable ignore the path
+ ctx.Logger().V(2).Info("skipping, following symlinks is not allowed", "path", path)
+ return nil
+ }
+
+ depth++
+
if depth > s.maxSymlinkDepth {
return errors.New("max symlink depth reached")
}
+
cleanPath := filepath.Clean(path)
resolvedPath, err := os.Readlink(cleanPath)
@@ -196,7 +191,7 @@ func (s *Source) scanSymlink(
"resolvedPath", resolvedPath,
"depth", depth,
)
- return s.scanSymlink(ctx, chunksChan, workerPool, rootPath, depth+1, resolvedPath)
+ return s.scanSymlink(ctx, chunksChan, rootPath, depth, resolvedPath)
}
if fileInfo.IsDir() {
@@ -207,7 +202,7 @@ func (s *Source) scanSymlink(
"depth", depth,
)
- return s.scanDir(ctx, chunksChan, workerPool, rootPath, depth+1, resolvedPath)
+ return s.scanDir(ctx, chunksChan, rootPath, depth, resolvedPath)
}
ctx.Logger().V(5).Info(
"found symlink to file",
@@ -223,25 +218,20 @@ func (s *Source) scanSymlink(
// Resume checks are handled by the calling scanDir function.
resumptionKey := rootPath
- workerPool.Go(func() error {
- if !fileInfo.Mode().Type().IsRegular() {
- ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath)
- return nil
- }
- if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil {
- ctx.Logger().Error(err, "error scanning file", "path", resolvedPath)
- }
- s.SetEncodedResumeInfoFor(resumptionKey, cleanPath)
+ if !fileInfo.Mode().Type().IsRegular() {
+ ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath)
return nil
- })
-
+ }
+ if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil {
+ ctx.Logger().Error(err, "error scanning file", "path", resolvedPath)
+ }
+ s.SetEncodedResumeInfoFor(resumptionKey, cleanPath)
return nil
}
func (s *Source) scanDir(
ctx trContext.Context,
chunksChan chan *sources.Chunk,
- workerPool *errgroup.Group,
rootPath string,
depth int,
path string,
@@ -285,6 +275,9 @@ func (s *Source) scanDir(
return fmt.Errorf("readdir error: %w", err)
}
+ workerPool := new(errgroup.Group)
+ workerPool.SetLimit(s.concurrency)
+
for _, entry := range entries {
entryPath := filepath.Join(path, entry.Name())
if s.filter != nil && !s.filter.Pass(entryPath) {
@@ -308,7 +301,7 @@ func (s *Source) scanDir(
// traverse into it to find where to resume.
if entry.IsDir() && strings.HasPrefix(resumeAfter, entryPath+string(filepath.Separator)) {
// Recurse into this directory to find the resume point.
- if err := s.scanDir(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {
+ if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning directory", "path", entryPath)
}
// After recursing, clear local resumeAfter. The child scanDir will have
@@ -323,17 +316,12 @@ func (s *Source) scanDir(
if entry.Type()&os.ModeSymlink != 0 {
ctx.Logger().V(5).Info("Entry found is a symlink", "path", entryPath)
- if !s.canFollowSymlinks() {
- // If the file or directory is a symlink but the followSymlinks is disable ignore the path
- ctx.Logger().Info("skipping, following symlinks is not allowed", "path", entryPath)
- continue
- }
- if err := s.scanSymlink(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {
+ if err := s.scanSymlink(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning symlink", "path", entryPath)
}
} else if entry.IsDir() {
ctx.Logger().V(5).Info("Entry found is a directory", "path", entryPath)
- if err := s.scanDir(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {
+ if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning directory", "path", entryPath)
}
} else {
@@ -351,20 +339,18 @@ func (s *Source) scanDir(
}
}
+ _ = workerPool.Wait() // [TODO] Handle errors
+
return nil
}
-var skipSymlinkErr = errors.New("skipping symlink")
-
func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, path string) error {
fileCtx := trContext.WithValues(ctx, "path", path)
- fileStat, err := os.Lstat(path)
+
+ _, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("unable to stat file: %w", err)
}
- if fileStat.Mode()&os.ModeSymlink != 0 {
- return skipSymlinkErr
- }
// Check if file is binary and should be skipped
if (s.skipBinaries || feature.ForceSkipBinaries.Load()) && common.IsBinary(path) {
@@ -435,28 +421,17 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor
go func() {
defer close(ch)
if fileInfo.Mode()&os.ModeSymlink != 0 {
- if !s.canFollowSymlinks() {
- // If the file or directory is a symlink but the followSymlinks is disable ignore the path
- logger.Info("skipping, following symlinks is not allowed", "path", cleanPath)
- return
- }
// if the root path is a symlink we scan the symlink
ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(s.concurrency)
- initialDepth := 1
- scanErr = s.scanSymlink(ctx, ch, workerPool, rootPath, initialDepth, cleanPath)
- _ = workerPool.Wait()
+ initialDepth := 0
+ scanErr = s.scanSymlink(ctx, ch, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else if fileInfo.IsDir() {
ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(s.concurrency)
- initialDepth := 1
+ initialDepth := 0
// TODO: Finer grain error tracking of individual chunks.
- scanErr = s.scanDir(ctx, ch, workerPool, rootPath, initialDepth, cleanPath)
- _ = workerPool.Wait()
+ scanErr = s.scanDir(ctx, ch, rootPath, initialDepth, cleanPath)
s.ClearEncodedResumeInfoFor(rootPath)
} else {
ctx.Logger().V(5).Info("Root path is a file", "path", cleanPath)
@@ -480,9 +455,7 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor
}
if scanErr != nil && !errors.Is(scanErr, io.EOF) {
- if !errors.Is(scanErr, skipSymlinkErr) {
- logger.Error(scanErr, "error scanning filesystem")
- }
+ logger.Error(scanErr, "error scanning filesystem")
return reporter.ChunkErr(ctx, scanErr)
}
return nil
diff --git a/pkg/sources/filesystem/filesystem_symlink_test.go b/pkg/sources/filesystem/filesystem_symlink_test.go
index 6af5bbcba..f382fe4ba 100644
--- a/pkg/sources/filesystem/filesystem_symlink_test.go
+++ b/pkg/sources/filesystem/filesystem_symlink_test.go
@@ -8,7 +8,6 @@ import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
- "golang.org/x/sync/errgroup"
"google.golang.org/protobuf/types/known/anypb"
trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -506,10 +505,8 @@ func TestScanSymlink_NoError(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
go func() {
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(src.concurrency)
- err := src.scanSymlink(ctx, chunks, workerPool, filepath.Join(baseDir, "A"), 1, filepath.Join(baseDir, "A"))
- _ = workerPool.Wait()
+ path := filepath.Join(baseDir, "A")
+ err := src.scanSymlink(ctx, chunks, path, 0, path)
require.NoError(t, err)
close(chunks)
}()
@@ -555,19 +552,11 @@ func TestScanSymlink_MaxDepthExceeded(t *testing.T) {
maxSymlinkDepth: 2,
}
chunks := make(chan *sources.Chunk, 10)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(src.concurrency)
-
- err = src.scanSymlink(
- ctx,
- chunks,
- workerPool,
- filepath.Join(baseDir, "A"),
- 1,
- filepath.Join(baseDir, "A"),
- )
- _ = workerPool.Wait()
+
+ path := filepath.Join(baseDir, "A")
+ err = src.scanSymlink(ctx, chunks, path, 0, path)
close(chunks)
+
require.Error(t, err)
require.EqualError(t, err, "max symlink depth reached")
}
@@ -597,18 +586,8 @@ func TestScanSymlink_FileTarget(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(src.concurrency)
-
- err = src.scanSymlink(
- ctx,
- chunks,
- workerPool,
- symlinkPath,
- 1,
- symlinkPath,
- )
- _ = workerPool.Wait()
+
+ err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)
require.NoError(t, err)
close(chunks)
var chunkCount int
@@ -639,18 +618,8 @@ func TestScanSymlink_SelfLoop(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(src.concurrency)
-
- err = src.scanSymlink(
- ctx,
- chunks,
- workerPool,
- symlinkPath,
- 1,
- symlinkPath,
- )
- _ = workerPool.Wait()
+
+ err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)
close(chunks)
require.Error(t, err)
require.EqualError(t, err, "max symlink depth reached")
@@ -675,18 +644,8 @@ func TestScanSymlink_BrokenSymlink(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(src.concurrency)
-
- err = src.scanSymlink(
- ctx,
- chunks,
- workerPool,
- symlinkPath,
- 0,
- symlinkPath,
- )
- _ = workerPool.Wait()
+
+ err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)
close(chunks)
require.Error(t, err)
require.Contains(t, err.Error(), "lstat error")
@@ -714,18 +673,8 @@ func TestScanSymlink_TwoFileLoop(t *testing.T) {
}
chunks := make(chan *sources.Chunk, 10)
- workerPool := new(errgroup.Group)
- workerPool.SetLimit(src.concurrency)
-
- err = src.scanSymlink(
- ctx,
- chunks,
- workerPool,
- fileA,
- 0,
- fileA,
- )
- _ = workerPool.Wait()
+
+ err = src.scanSymlink(ctx, chunks, fileA, 0, fileA)
close(chunks)
require.Error(t, err)
require.EqualError(t, err, "max symlink depth reached")