Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Confine symlink state handling to scanSymlink in Filesystem source (#4807)
Charlie Gunyonafd5336caad0
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex 0760fccee..5aced6666 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -125,26 +125,15 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _}if fileInfo.Mode()&os.ModeSymlink != 0 {- if !s.canFollowSymlinks() {- // If the file or directory is a symlink but the followSymlinks is disable ignore the path- logger.Info("skipping, following symlinks is not allowed", "path", cleanPath)- continue- }// if the root path is a symlink we scan the symlinkctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)- workerPool := new(errgroup.Group)- workerPool.SetLimit(s.concurrency)- initialDepth := 1- err = s.scanSymlink(ctx, chunksChan, workerPool, rootPath, initialDepth, cleanPath)- _ = workerPool.Wait()+ initialDepth := 0+ err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanPath)s.ClearEncodedResumeInfoFor(rootPath)} else if fileInfo.IsDir() {ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)- workerPool := new(errgroup.Group)- workerPool.SetLimit(s.concurrency)- initialDepth := 1- err = s.scanDir(ctx, chunksChan, workerPool, rootPath, initialDepth, cleanPath)- _ = workerPool.Wait()+ initialDepth := 0+ err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanPath)s.ClearEncodedResumeInfoFor(rootPath)} else {if !fileInfo.Mode().IsRegular() {@@ -156,9 +145,7 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _}if err != nil && !errors.Is(err, io.EOF) {- if !errors.Is(err, skipSymlinkErr) {- logger.Error(err, "error scanning filesystem")- }+ logger.Error(err, "error scanning filesystem")}}@@ -168,14 +155,22 @@ func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _func (s *Source) scanSymlink(ctx trContext.Context,chunksChan chan *sources.Chunk,- workerPool *errgroup.Group,rootPath string,depth int,path string,) error {+ if !s.canFollowSymlinks() {+ // If the file or directory is a symlink but the followSymlinks is disable ignore the path+ ctx.Logger().V(2).Info("skipping, following symlinks is not allowed", "path", path)+ return nil+ }++ depth+++if depth > s.maxSymlinkDepth {return errors.New("max symlink depth reached")}+cleanPath := filepath.Clean(path)resolvedPath, err := os.Readlink(cleanPath)@@ -196,7 +191,7 @@ func (s *Source) scanSymlink("resolvedPath", resolvedPath,"depth", depth,)- return s.scanSymlink(ctx, chunksChan, workerPool, rootPath, depth+1, resolvedPath)+ return s.scanSymlink(ctx, chunksChan, rootPath, depth, resolvedPath)}if fileInfo.IsDir() {@@ -207,7 +202,7 @@ func (s *Source) scanSymlink("depth", depth,)- return s.scanDir(ctx, chunksChan, workerPool, rootPath, depth+1, resolvedPath)+ return s.scanDir(ctx, chunksChan, rootPath, depth, resolvedPath)}ctx.Logger().V(5).Info("found symlink to file",@@ -223,25 +218,20 @@ func (s *Source) scanSymlink(// Resume checks are handled by the calling scanDir function.resumptionKey := rootPath- workerPool.Go(func() error {- if !fileInfo.Mode().Type().IsRegular() {- ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath)- return nil- }- if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil {- ctx.Logger().Error(err, "error scanning file", "path", resolvedPath)- }- s.SetEncodedResumeInfoFor(resumptionKey, cleanPath)+ if !fileInfo.Mode().Type().IsRegular() {+ ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath)return nil- })-+ }+ if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil {+ ctx.Logger().Error(err, "error scanning file", "path", resolvedPath)+ }+ s.SetEncodedResumeInfoFor(resumptionKey, cleanPath)return nil}func (s *Source) scanDir(ctx trContext.Context,chunksChan chan *sources.Chunk,- workerPool *errgroup.Group,rootPath string,depth int,path string,@@ -285,6 +275,9 @@ func (s *Source) scanDir(return fmt.Errorf("readdir error: %w", err)}+ workerPool := new(errgroup.Group)+ workerPool.SetLimit(s.concurrency)+for _, entry := range entries {entryPath := filepath.Join(path, entry.Name())if s.filter != nil && !s.filter.Pass(entryPath) {@@ -308,7 +301,7 @@ func (s *Source) scanDir(// traverse into it to find where to resume.if entry.IsDir() && strings.HasPrefix(resumeAfter, entryPath+string(filepath.Separator)) {// Recurse into this directory to find the resume point.- if err := s.scanDir(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {+ if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {ctx.Logger().Error(err, "error scanning directory", "path", entryPath)}// After recursing, clear local resumeAfter. The child scanDir will have@@ -323,17 +316,12 @@ func (s *Source) scanDir(if entry.Type()&os.ModeSymlink != 0 {ctx.Logger().V(5).Info("Entry found is a symlink", "path", entryPath)- if !s.canFollowSymlinks() {- // If the file or directory is a symlink but the followSymlinks is disable ignore the path- ctx.Logger().Info("skipping, following symlinks is not allowed", "path", entryPath)- continue- }- if err := s.scanSymlink(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {+ if err := s.scanSymlink(ctx, chunksChan, rootPath, depth, entryPath); err != nil {ctx.Logger().Error(err, "error scanning symlink", "path", entryPath)}} else if entry.IsDir() {ctx.Logger().V(5).Info("Entry found is a directory", "path", entryPath)- if err := s.scanDir(ctx, chunksChan, workerPool, rootPath, depth, entryPath); err != nil {+ if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {ctx.Logger().Error(err, "error scanning directory", "path", entryPath)}} else {@@ -351,20 +339,18 @@ func (s *Source) scanDir(}}+ _ = workerPool.Wait() // [TODO] Handle errors+return nil}-var skipSymlinkErr = errors.New("skipping symlink")-func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, path string) error {fileCtx := trContext.WithValues(ctx, "path", path)- fileStat, err := os.Lstat(path)++ _, err := os.Lstat(path)if err != nil {return fmt.Errorf("unable to stat file: %w", err)}- if fileStat.Mode()&os.ModeSymlink != 0 {- return skipSymlinkErr- }// Check if file is binary and should be skippedif (s.skipBinaries || feature.ForceSkipBinaries.Load()) && common.IsBinary(path) {@@ -435,28 +421,17 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, reporgo func() {defer close(ch)if fileInfo.Mode()&os.ModeSymlink != 0 {- if !s.canFollowSymlinks() {- // If the file or directory is a symlink but the followSymlinks is disable ignore the path- logger.Info("skipping, following symlinks is not allowed", "path", cleanPath)- return- }// if the root path is a symlink we scan the symlinkctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)- workerPool := new(errgroup.Group)- workerPool.SetLimit(s.concurrency)- initialDepth := 1- scanErr = s.scanSymlink(ctx, ch, workerPool, rootPath, initialDepth, cleanPath)- _ = workerPool.Wait()+ initialDepth := 0+ scanErr = s.scanSymlink(ctx, ch, rootPath, initialDepth, cleanPath)s.ClearEncodedResumeInfoFor(rootPath)} else if fileInfo.IsDir() {ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)- workerPool := new(errgroup.Group)- workerPool.SetLimit(s.concurrency)- initialDepth := 1+ initialDepth := 0// TODO: Finer grain error tracking of individual chunks.- scanErr = s.scanDir(ctx, ch, workerPool, rootPath, initialDepth, cleanPath)- _ = workerPool.Wait()+ scanErr = s.scanDir(ctx, ch, rootPath, initialDepth, cleanPath)s.ClearEncodedResumeInfoFor(rootPath)} else {ctx.Logger().V(5).Info("Root path is a file", "path", cleanPath)@@ -480,9 +455,7 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor}if scanErr != nil && !errors.Is(scanErr, io.EOF) {- if !errors.Is(scanErr, skipSymlinkErr) {- logger.Error(scanErr, "error scanning filesystem")- }+ logger.Error(scanErr, "error scanning filesystem")return reporter.ChunkErr(ctx, scanErr)}return nildiff --git a/pkg/sources/filesystem/filesystem_symlink_test.go b/pkg/sources/filesystem/filesystem_symlink_test.goindex 6af5bbcba..f382fe4ba 100644--- a/pkg/sources/filesystem/filesystem_symlink_test.go+++ b/pkg/sources/filesystem/filesystem_symlink_test.go@@ -8,7 +8,6 @@ import ("github.com/stretchr/testify/assert""github.com/stretchr/testify/require"- "golang.org/x/sync/errgroup""google.golang.org/protobuf/types/known/anypb"trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -506,10 +505,8 @@ func TestScanSymlink_NoError(t *testing.T) {}chunks := make(chan *sources.Chunk, 10)go func() {- workerPool := new(errgroup.Group)- workerPool.SetLimit(src.concurrency)- err := src.scanSymlink(ctx, chunks, workerPool, filepath.Join(baseDir, "A"), 1, filepath.Join(baseDir, "A"))- _ = workerPool.Wait()+ path := filepath.Join(baseDir, "A")+ err := src.scanSymlink(ctx, chunks, path, 0, path)require.NoError(t, err)close(chunks)}()@@ -555,19 +552,11 @@ func TestScanSymlink_MaxDepthExceeded(t *testing.T) {maxSymlinkDepth: 2,}chunks := make(chan *sources.Chunk, 10)- workerPool := new(errgroup.Group)- workerPool.SetLimit(src.concurrency)-- err = src.scanSymlink(- ctx,- chunks,- workerPool,- filepath.Join(baseDir, "A"),- 1,- filepath.Join(baseDir, "A"),- )- _ = workerPool.Wait()++ path := filepath.Join(baseDir, "A")+ err = src.scanSymlink(ctx, chunks, path, 0, path)close(chunks)+require.Error(t, err)require.EqualError(t, err, "max symlink depth reached")}@@ -597,18 +586,8 @@ func TestScanSymlink_FileTarget(t *testing.T) {}chunks := make(chan *sources.Chunk, 10)- workerPool := new(errgroup.Group)- workerPool.SetLimit(src.concurrency)-- err = src.scanSymlink(- ctx,- chunks,- workerPool,- symlinkPath,- 1,- symlinkPath,- )- _ = workerPool.Wait()++ err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)require.NoError(t, err)close(chunks)var chunkCount int@@ -639,18 +618,8 @@ func TestScanSymlink_SelfLoop(t *testing.T) {}chunks := make(chan *sources.Chunk, 10)- workerPool := new(errgroup.Group)- workerPool.SetLimit(src.concurrency)-- err = src.scanSymlink(- ctx,- chunks,- workerPool,- symlinkPath,- 1,- symlinkPath,- )- _ = workerPool.Wait()++ err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)close(chunks)require.Error(t, err)require.EqualError(t, err, "max symlink depth reached")@@ -675,18 +644,8 @@ func TestScanSymlink_BrokenSymlink(t *testing.T) {}chunks := make(chan *sources.Chunk, 10)- workerPool := new(errgroup.Group)- workerPool.SetLimit(src.concurrency)-- err = src.scanSymlink(- ctx,- chunks,- workerPool,- symlinkPath,- 0,- symlinkPath,- )- _ = workerPool.Wait()++ err = src.scanSymlink(ctx, chunks, symlinkPath, 0, symlinkPath)close(chunks)require.Error(t, err)require.Contains(t, err.Error(), "lstat error")@@ -714,18 +673,8 @@ func TestScanSymlink_TwoFileLoop(t *testing.T) {}chunks := make(chan *sources.Chunk, 10)- workerPool := new(errgroup.Group)- workerPool.SetLimit(src.concurrency)-- err = src.scanSymlink(- ctx,- chunks,- workerPool,- fileA,- 0,- fileA,- )- _ = workerPool.Wait()++ err = src.scanSymlink(ctx, chunks, fileA, 0, fileA)close(chunks)require.Error(t, err)require.EqualError(t, err, "max symlink depth reached")