Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Allow filesystem exclusion to eagerly prune the enumerated tree (#4008)
Kashif Khande874d31bd3b
diff --git a/pkg/common/filter.go b/pkg/common/filter.goindex e5bbcd251..8462ccac0 100644--- a/pkg/common/filter.go+++ b/pkg/common/filter.go@@ -97,8 +97,10 @@ func (filter *Filter) Pass(object string) bool {if filter == nil {return true}+excluded := filter.exclude.Matches(object)included := filter.include.Matches(object)+return !excluded && included}@@ -114,3 +116,8 @@ func (rules *FilterRuleSet) Matches(object string) bool {}return false}++// ShouldExclude return true if any regular expressions in the exclude FilterRuleSet matches the path.+func (filter *Filter) ShouldExclude(path string) bool {+ return filter.exclude.Matches(path)+}diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex 98805882d..994ccd76c 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -127,17 +127,25 @@ func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sourctx.Logger().Error(err, "error walking directory")return nil}+fullPath := filepath.Join(path, relativePath)+ // check if the full path is not matching any pattern in include FilterRuleSet and matching any exclude FilterRuleSet.+ if s.filter != nil && !s.filter.Pass(fullPath) {+ // skip excluded directories+ if d.IsDir() && s.filter.ShouldExclude(fullPath) {+ return fs.SkipDir+ }++ return nil // skip the file+ }+// Skip over non-regular files. We do this check here to suppress noisy// logs for trying to scan directories and other non-regular files in// our traversal.if !d.Type().IsRegular() {return nil}- if s.filter != nil && !s.filter.Pass(fullPath) {- return nil- }workerPool.Go(func() error {if err = s.scanFile(ctx, fullPath, chunksChan); err != nil {diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.goindex a5ce36122..f7adc0916 100644--- a/pkg/sources/filesystem/filesystem_test.go+++ b/pkg/sources/filesystem/filesystem_test.go@@ -10,6 +10,7 @@ import ("github.com/go-logr/logr""github.com/kylelemons/godebug/pretty""github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require""google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -300,6 +301,83 @@ func TestChunkUnitReporterErr(t *testing.T) {assert.Error(t, err)}+func TestSkipDir(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ // create a temp directory with files+ ignoreDir, cleanupDir, err := createTempDir("", "ignore1", "ignore2", "ignore3")+ require.NoError(t, err)+ defer cleanupDir()++ // create an ExcludePathsFile that contains the ignoreDir path+ excludeFile, cleanupFile, err := createTempFile("", ignoreDir+"\n")+ require.NoError(t, err)+ defer cleanupFile()++ conn, err := anypb.New(&sourcespb.Filesystem{+ ExcludePathsFile: excludeFile.Name(),+ })+ require.NoError(t, err)++ // initialize the source.+ s := Source{}+ err = s.Init(ctx, "exclude directory", 0, 0, true, conn, 1)+ require.NoError(t, err)++ reporter := sourcestest.TestReporter{}+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{+ ID: ignoreDir,+ }, &reporter)+ require.NoError(t, err)++ require.Equal(t, 0, len(reporter.Chunks), "Expected no chunks from excluded directory")+ require.Equal(t, 0, len(reporter.ChunkErrs), "Expected no errors for excluded directory")+}++func TestScanSubDirFile(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ // create a temp directory with files+ parentDir, cleanupParentDir, err := createTempDir("", "file1")+ require.NoError(t, err)+ defer cleanupParentDir()++ childDir, cleanupChildDir, err := createTempDir(parentDir, "file2")+ require.NoError(t, err)+ defer cleanupChildDir()++ // create a file in child directory+ file, cleanupFile, err := createTempFile(childDir, "should scan this file")+ require.NoError(t, err)+ defer cleanupFile()++ // create an IncludePathsFile that contains the file path+ includeFile, cleanupFile, err := createTempFile("", file.Name()+"\n")+ require.NoError(t, err)+ defer cleanupFile()++ conn, err := anypb.New(&sourcespb.Filesystem{+ IncludePathsFile: includeFile.Name(),+ })+ require.NoError(t, err)++ // initialize the source.+ s := Source{}+ err = s.Init(ctx, "include sub directory file", 0, 0, true, conn, 1)+ require.NoError(t, err)++ reporter := sourcestest.TestReporter{}+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{+ ID: parentDir,+ }, &reporter)+ require.NoError(t, err)++ require.Equal(t, 1, len(reporter.Chunks), "Expected chunks from included file")+ require.Equal(t, 0, len(reporter.ChunkErrs), "Expected no errors")+}+// createTempFile is a helper function to create a temporary file in the given// directory with the provided contents. If dir is "", the operating system's// temp directory is used.