Skip to content

Find, verify, and analyze leaked credentials

Allow filesystem exclusion to eagerly prune the enumerated tree (#4008)

Kashif Khande874d31bd3b
diff --git a/pkg/common/filter.go b/pkg/common/filter.go
index e5bbcd251..8462ccac0 100644
--- a/pkg/common/filter.go
+++ b/pkg/common/filter.go
@@ -97,8 +97,10 @@ func (filter *Filter) Pass(object string) bool {
if filter == nil {
return true
}
+
excluded := filter.exclude.Matches(object)
included := filter.include.Matches(object)
+
return !excluded && included
}
@@ -114,3 +116,8 @@ func (rules *FilterRuleSet) Matches(object string) bool {
}
return false
}
+
+// ShouldExclude return true if any regular expressions in the exclude FilterRuleSet matches the path.
+func (filter *Filter) ShouldExclude(path string) bool {
+ return filter.exclude.Matches(path)
+}
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index 98805882d..994ccd76c 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -127,17 +127,25 @@ func (s *Source) scanDir(ctx context.Context, path string, chunksChan chan *sour
ctx.Logger().Error(err, "error walking directory")
return nil
}
+
fullPath := filepath.Join(path, relativePath)
+ // check if the full path is not matching any pattern in include FilterRuleSet and matching any exclude FilterRuleSet.
+ if s.filter != nil && !s.filter.Pass(fullPath) {
+ // skip excluded directories
+ if d.IsDir() && s.filter.ShouldExclude(fullPath) {
+ return fs.SkipDir
+ }
+
+ return nil // skip the file
+ }
+
// Skip over non-regular files. We do this check here to suppress noisy
// logs for trying to scan directories and other non-regular files in
// our traversal.
if !d.Type().IsRegular() {
return nil
}
- if s.filter != nil && !s.filter.Pass(fullPath) {
- return nil
- }
workerPool.Go(func() error {
if err = s.scanFile(ctx, fullPath, chunksChan); err != nil {
diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.go
index a5ce36122..f7adc0916 100644
--- a/pkg/sources/filesystem/filesystem_test.go
+++ b/pkg/sources/filesystem/filesystem_test.go
@@ -10,6 +10,7 @@ import (
"github.com/go-logr/logr"
"github.com/kylelemons/godebug/pretty"
"github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
"google.golang.org/protobuf/types/known/anypb"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
@@ -300,6 +301,83 @@ func TestChunkUnitReporterErr(t *testing.T) {
assert.Error(t, err)
}
+func TestSkipDir(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+
+ // create a temp directory with files
+ ignoreDir, cleanupDir, err := createTempDir("", "ignore1", "ignore2", "ignore3")
+ require.NoError(t, err)
+ defer cleanupDir()
+
+ // create an ExcludePathsFile that contains the ignoreDir path
+ excludeFile, cleanupFile, err := createTempFile("", ignoreDir+"\n")
+ require.NoError(t, err)
+ defer cleanupFile()
+
+ conn, err := anypb.New(&sourcespb.Filesystem{
+ ExcludePathsFile: excludeFile.Name(),
+ })
+ require.NoError(t, err)
+
+ // initialize the source.
+ s := Source{}
+ err = s.Init(ctx, "exclude directory", 0, 0, true, conn, 1)
+ require.NoError(t, err)
+
+ reporter := sourcestest.TestReporter{}
+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
+ ID: ignoreDir,
+ }, &reporter)
+ require.NoError(t, err)
+
+ require.Equal(t, 0, len(reporter.Chunks), "Expected no chunks from excluded directory")
+ require.Equal(t, 0, len(reporter.ChunkErrs), "Expected no errors for excluded directory")
+}
+
+func TestScanSubDirFile(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+
+ // create a temp directory with files
+ parentDir, cleanupParentDir, err := createTempDir("", "file1")
+ require.NoError(t, err)
+ defer cleanupParentDir()
+
+ childDir, cleanupChildDir, err := createTempDir(parentDir, "file2")
+ require.NoError(t, err)
+ defer cleanupChildDir()
+
+ // create a file in child directory
+ file, cleanupFile, err := createTempFile(childDir, "should scan this file")
+ require.NoError(t, err)
+ defer cleanupFile()
+
+ // create an IncludePathsFile that contains the file path
+ includeFile, cleanupFile, err := createTempFile("", file.Name()+"\n")
+ require.NoError(t, err)
+ defer cleanupFile()
+
+ conn, err := anypb.New(&sourcespb.Filesystem{
+ IncludePathsFile: includeFile.Name(),
+ })
+ require.NoError(t, err)
+
+ // initialize the source.
+ s := Source{}
+ err = s.Init(ctx, "include sub directory file", 0, 0, true, conn, 1)
+ require.NoError(t, err)
+
+ reporter := sourcestest.TestReporter{}
+ err = s.ChunkUnit(ctx, sources.CommonSourceUnit{
+ ID: parentDir,
+ }, &reporter)
+ require.NoError(t, err)
+
+ require.Equal(t, 1, len(reporter.Chunks), "Expected chunks from included file")
+ require.Equal(t, 0, len(reporter.ChunkErrs), "Expected no errors")
+}
+
// createTempFile is a helper function to create a temporary file in the given
// directory with the provided contents. If dir is "", the operating system's
// temp directory is used.