Skip to content

Find, verify, and analyze leaked credentials

Clean up Filesystem source logging for per-source debug (#5465)

Kashif Khand6ea4a52e3f4
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.go
index 9e6206423..9b5d21ef6 100644
--- a/pkg/sources/filesystem/filesystem.go
+++ b/pkg/sources/filesystem/filesystem.go
@@ -7,9 +7,9 @@ import (
"path/filepath"
"slices"
"strings"
+ "time"
"github.com/go-errors/errors"
- "github.com/go-logr/logr"
"golang.org/x/sync/errgroup"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"
@@ -33,7 +33,6 @@ type Source struct {
concurrency int
verify bool
paths []string
- log logr.Logger
filter *common.Filter
skipBinaries bool
sources.Progress
@@ -67,8 +66,6 @@ func (s *Source) JobID() sources.JobID {
// Init returns an initialized Filesystem source.
func (s *Source) Init(aCtx trContext.Context, name string, jobId sources.JobID, sourceId sources.SourceID, verify bool, connection *anypb.Any, concurrency int) error {
- s.log = aCtx.Logger()
-
s.concurrency = concurrency
s.name = name
s.sourceId = sourceId
@@ -91,6 +88,14 @@ func (s *Source) Init(aCtx trContext.Context, name string, jobId sources.JobID,
if err != nil {
return err
}
+
+ aCtx.Logger().V(1).Info("filesystem source initialized",
+ "path_count", len(s.paths),
+ "skip_binaries", s.skipBinaries,
+ "max_symlink_depth", s.maxSymlinkDepth,
+ "include_paths_file", conn.IncludePathsFile,
+ "exclude_paths_file", conn.ExcludePathsFile,
+ )
return nil
}
@@ -114,47 +119,63 @@ func (s *Source) canFollowSymlinks() bool {
// Chunks emits chunks of bytes over a channel.
func (s *Source) Chunks(ctx trContext.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {
for i, rootPath := range s.paths {
- logger := ctx.Logger().WithValues("path", rootPath)
if common.IsDone(ctx) {
return nil
}
s.SetProgressComplete(i, len(s.paths), fmt.Sprintf("Path: %s", rootPath), s.GetProgress().EncodedResumeInfo)
- cleanPath := filepath.Clean(rootPath)
- fileInfo, err := os.Lstat(cleanPath)
+ // root_path is kept apart from path, which the lines below use for the item being scanned.
+ pathCtx := trContext.WithValues(ctx, "root_path", rootPath)
+ fileInfo, err := os.Lstat(filepath.Clean(rootPath))
if err != nil {
- logger.Error(err, "unable to get file info")
+ pathCtx.Logger().Error(err, "unable to get file info")
continue
}
- if fileInfo.Mode()&os.ModeSymlink != 0 {
- // if the root path is a symlink we scan the symlink
- ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)
- initialDepth := 0
- err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanPath)
- s.ClearEncodedResumeInfoFor(rootPath)
- } else if fileInfo.IsDir() {
- ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)
- initialDepth := 0
- err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanPath)
- s.ClearEncodedResumeInfoFor(rootPath)
- } else {
- if !fileInfo.Mode().IsRegular() {
- logger.Info("skipping non-regular file", "path", cleanPath)
- continue
- }
- ctx.Logger().V(5).Info("Root path is a file", "path", cleanPath)
- err = s.scanFile(ctx, chunksChan, cleanPath)
- }
-
- if err != nil && !errors.Is(err, io.EOF) {
- logger.Error(err, "error scanning filesystem")
+ if err := s.scanRootPath(pathCtx, chunksChan, rootPath, fileInfo); err != nil && !errors.Is(err, io.EOF) {
+ pathCtx.Logger().Error(err, "error scanning filesystem")
}
}
return nil
}
+// scanRootPath scans one configured path, which can be a symlink, a directory, or a file.
+// It is shared by Chunks and ChunkUnit so both scan and log a root path the same way.
+func (s *Source) scanRootPath(
+ ctx trContext.Context,
+ chunksChan chan *sources.Chunk,
+ rootPath string,
+ fileInfo os.FileInfo,
+) error {
+ cleanPath := filepath.Clean(rootPath)
+ initialDepth := 0
+ start := time.Now()
+
+ var err error
+ switch {
+ case fileInfo.Mode()&os.ModeSymlink != 0:
+ ctx.Logger().V(2).Info("scanning path", "type", "symlink")
+ err = s.scanSymlink(ctx, chunksChan, rootPath, initialDepth, cleanPath)
+ s.ClearEncodedResumeInfoFor(rootPath)
+ case fileInfo.IsDir():
+ ctx.Logger().V(2).Info("scanning path", "type", "dir")
+ err = s.scanDir(ctx, chunksChan, rootPath, initialDepth, cleanPath)
+ s.ClearEncodedResumeInfoFor(rootPath)
+ case !fileInfo.Mode().IsRegular():
+ // The user asked for this path, so tell them it was not scanned.
+ ctx.Logger().Info("skipping file", "reason", "not a regular file")
+ return nil
+ default:
+ ctx.Logger().V(2).Info("scanning path", "type", "file")
+ // TODO: Finer grain error tracking of individual chunks (in the case of archives).
+ err = s.scanFile(ctx, chunksChan, cleanPath)
+ }
+
+ ctx.Logger().V(2).Info("finished scanning path", "duration", time.Since(start).String())
+ return err
+}
+
func (s *Source) scanSymlink(
ctx trContext.Context,
chunksChan chan *sources.Chunk,
@@ -163,8 +184,8 @@ func (s *Source) scanSymlink(
path string,
) error {
if !s.canFollowSymlinks() {
- // If the file or directory is a symlink but the followSymlinks is disable ignore the path
- ctx.Logger().V(2).Info("skipping, following symlinks is not allowed", "path", path)
+ // Symlinks are only followed when a max symlink depth is set.
+ ctx.Logger().V(4).Info("skipping symlink", "path", path, "reason", "following symlinks is not enabled")
return nil
}
@@ -187,33 +208,32 @@ func (s *Source) scanSymlink(
if err != nil {
return fmt.Errorf("lstat error: %w", err)
}
- if fileInfo.Mode()&os.ModeSymlink != 0 {
- ctx.Logger().V(5).Info(
- "found symlink to symlink",
- "symlinkPath", cleanPath,
- "resolvedPath", resolvedPath,
- "depth", depth,
- )
- return s.scanSymlink(ctx, chunksChan, rootPath, depth, resolvedPath)
- }
- if fileInfo.IsDir() {
- ctx.Logger().V(5).Info(
- "found symlink to dir",
- "symlinkPath", cleanPath,
- "resolvedPath", resolvedPath,
- "depth", depth,
- )
-
- return s.scanDir(ctx, chunksChan, rootPath, depth, resolvedPath)
+ targetType := "other"
+ switch {
+ case fileInfo.Mode()&os.ModeSymlink != 0:
+ targetType = "symlink"
+ case fileInfo.IsDir():
+ targetType = "dir"
+ case fileInfo.Mode().IsRegular():
+ targetType = "file"
}
- ctx.Logger().V(5).Info(
- "found symlink to file",
- "symlinkPath", cleanPath,
- "resolvedPath", resolvedPath,
+ ctx.Logger().V(5).Info("resolved symlink",
+ "path", cleanPath,
+ "resolved_path", resolvedPath,
+ "target_type", targetType,
"depth", depth,
)
+
+ switch targetType {
+ case "symlink":
+ return s.scanSymlink(ctx, chunksChan, rootPath, depth, resolvedPath)
+ case "dir":
+ return s.scanDir(ctx, chunksChan, rootPath, depth, resolvedPath)
+ }
+
if s.filter != nil && !s.filter.Pass(resolvedPath) {
+ ctx.Logger().V(4).Info("skipping file", "path", resolvedPath, "reason", "excluded by path filter")
return nil
}
@@ -222,7 +242,7 @@ func (s *Source) scanSymlink(
resumptionKey := rootPath
if !fileInfo.Mode().Type().IsRegular() {
- ctx.Logger().V(5).Info("skipping non-regular file", "path", resolvedPath)
+ ctx.Logger().V(4).Info("skipping file", "path", resolvedPath, "reason", "not a regular file")
return nil
}
if err := s.scanFile(ctx, chunksChan, resolvedPath); err != nil {
@@ -261,6 +281,7 @@ func (s *Source) scanDir(
// check if the full path is not matching any pattern in include
// FilterRuleSet and matching any exclude FilterRuleSet.
if s.filter != nil && s.filter.ShouldExclude(path) {
+ ctx.Logger().V(4).Info("skipping directory", "path", path, "reason", "excluded by path filter")
return nil
}
@@ -283,6 +304,11 @@ func (s *Source) scanDir(
if comparePathsForResume(path, resumeAfter) < 0 {
// This directory comes before the resume point in traversal order,
// meaning it was already fully scanned. Skip it entirely.
+ ctx.Logger().V(4).Info("skipping directory",
+ "path", path,
+ "reason", "already scanned before resume point",
+ "resume_after", resumeAfter,
+ )
return nil
}
// This directory comes after the resume point, so we've already passed
@@ -290,20 +316,25 @@ func (s *Source) scanDir(
resumeAfter = ""
}
- ctx.Logger().V(5).Info("Full path found is", "fullPath", path)
-
entries, err := os.ReadDir(path)
if err != nil {
return fmt.Errorf("readdir error: %w", err)
}
+ ctx.Logger().V(3).Info("scanning directory", "path", path, "entry_count", len(entries))
+ if resumeAfter != "" {
+ // Entries up to the resume point are skipped without a line each, so log the resume point once.
+ ctx.Logger().V(4).Info("resuming scan in directory", "path", path, "resume_after", resumeAfter)
+ }
workerPool := new(errgroup.Group)
workerPool.SetLimit(s.concurrency)
for _, entry := range entries {
entryPath := filepath.Join(path, entry.Name())
+ // Directories and symlinks are not skipped here because a path below them can still pass the filter.
if s.filter != nil && !s.filter.Pass(entryPath) {
if !entry.IsDir() && entry.Type()&os.ModeSymlink == 0 {
+ ctx.Logger().V(4).Info("skipping file", "path", entryPath, "reason", "excluded by path filter")
continue
}
}
@@ -316,6 +347,7 @@ func (s *Source) scanDir(
if resumeAfter != "" {
// If this entry is the resume point, stop skipping.
if entryPath == resumeAfter {
+ ctx.Logger().V(4).Info("found resume point, continuing scan after it", "path", entryPath)
resumeAfter = ""
continue // Skip the resume point itself since it was already processed.
}
@@ -337,20 +369,18 @@ func (s *Source) scanDir(
}
if entry.Type()&os.ModeSymlink != 0 {
- ctx.Logger().V(5).Info("Entry found is a symlink", "path", entryPath)
if err := s.scanSymlink(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning symlink", "path", entryPath)
}
} else if entry.IsDir() {
- ctx.Logger().V(5).Info("Entry found is a directory", "path", entryPath)
if err := s.scanDir(ctx, chunksChan, rootPath, depth, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning directory", "path", entryPath)
}
} else {
if !entry.Type().IsRegular() {
+ ctx.Logger().V(4).Info("skipping file", "path", entryPath, "reason", "not a regular file")
continue
}
- ctx.Logger().V(5).Info("Entry found is a file", "path", entryPath)
workerPool.Go(func() error {
if err := s.scanFile(ctx, chunksChan, entryPath); err != nil {
ctx.Logger().Error(err, "error scanning file", "path", entryPath)
@@ -369,14 +399,14 @@ func (s *Source) scanDir(
func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk, path string) error {
fileCtx := trContext.WithValues(ctx, "path", path)
- _, err := os.Lstat(path)
+ fileInfo, err := os.Lstat(path)
if err != nil {
return fmt.Errorf("unable to stat file: %w", err)
}
// Check if file is binary and should be skipped
if (s.skipBinaries || feature.ForceSkipBinaries.Load()) && common.IsBinary(path) {
- fileCtx.Logger().V(5).Info("skipping binary file", "path", path)
+ fileCtx.Logger().V(4).Info("skipping file", "reason", "binary files are skipped")
return nil
}
@@ -386,7 +416,8 @@ func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk,
}
defer func() { _ = inputFile.Close() }()
- fileCtx.Logger().V(3).Info("scanning file")
+ fileCtx.Logger().V(3).Info("scanning file", "size", fileInfo.Size())
+ start := time.Now()
chunkSkel := &sources.Chunk{
SourceType: s.Type(),
@@ -403,7 +434,9 @@ func (s *Source) scanFile(ctx trContext.Context, chunksChan chan *sources.Chunk,
SourceVerify: s.verify,
}
- return handlers.HandleFile(fileCtx, inputFile, chunkSkel, sources.ChanReporter{Ch: chunksChan})
+ err = handlers.HandleFile(fileCtx, inputFile, chunkSkel, sources.ChanReporter{Ch: chunksChan})
+ fileCtx.Logger().V(5).Info("finished scanning file", "duration", time.Since(start).String())
+ return err
}
// Enumerate implements SourceUnitEnumerator interface. This implementation simply
@@ -429,42 +462,20 @@ func (s *Source) Enumerate(ctx trContext.Context, reporter sources.UnitReporter)
// ChunkUnit implements SourceUnitChunker interface.
func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, reporter sources.ChunkReporter) error {
rootPath, _ := unit.SourceUnitID()
- logger := ctx.Logger().WithValues("path", rootPath)
+ ctx = trContext.WithValues(ctx, "root_path", rootPath)
- cleanPath := filepath.Clean(rootPath)
- fileInfo, err := os.Lstat(cleanPath)
+ // This will always be the FileInfo we use to decide dir vs file
+ fileInfo, err := os.Lstat(filepath.Clean(rootPath))
if err != nil {
return reporter.ChunkErr(ctx, fmt.Errorf("unable to get file info: %w", err))
}
- // This will always be the FileInfo we use to decide dir vs file
ch := make(chan *sources.Chunk)
var scanErr error
go func() {
defer close(ch)
- if fileInfo.Mode()&os.ModeSymlink != 0 {
- // if the root path is a symlink we scan the symlink
- ctx.Logger().V(5).Info("Root path is a symlink", "path", cleanPath)
- initialDepth := 0
- scanErr = s.scanSymlink(ctx, ch, rootPath, initialDepth, cleanPath)
- s.ClearEncodedResumeInfoFor(rootPath)
-
- } else if fileInfo.IsDir() {
- ctx.Logger().V(5).Info("Root path is a dir", "path", cleanPath)
- initialDepth := 0
- // TODO: Finer grain error tracking of individual chunks.
- scanErr = s.scanDir(ctx, ch, rootPath, initialDepth, cleanPath)
- s.ClearEncodedResumeInfoFor(rootPath)
- } else {
- ctx.Logger().V(5).Info("Root path is a file", "path", cleanPath)
- // TODO: Finer grain error tracking of individual
- // chunks (in the case of archives).
- if !fileInfo.Mode().IsRegular() {
- logger.Info("skipping non-regular file", "path", cleanPath)
- return
- }
- scanErr = s.scanFile(ctx, ch, cleanPath)
- }
+ // TODO: Finer grain error tracking of individual chunks.
+ scanErr = s.scanRootPath(ctx, ch, rootPath, fileInfo)
}()
for chunk := range ch {
@@ -477,7 +488,7 @@ func (s *Source) ChunkUnit(ctx trContext.Context, unit sources.SourceUnit, repor
}
if scanErr != nil && !errors.Is(scanErr, io.EOF) {
- logger.Error(scanErr, "error scanning filesystem")
+ ctx.Logger().Error(scanErr, "error scanning filesystem")
return reporter.ChunkErr(ctx, scanErr)
}
return nil
diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.go
index 67b5e0c4c..a61f24b01 100644
--- a/pkg/sources/filesystem/filesystem_test.go
+++ b/pkg/sources/filesystem/filesystem_test.go
@@ -11,13 +11,18 @@ import (
"time"
"github.com/go-logr/logr"
+ "github.com/go-logr/zapr"
"github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
+ "go.uber.org/zap"
+ "go.uber.org/zap/zapcore"
+ "go.uber.org/zap/zaptest/observer"
"google.golang.org/protobuf/testing/protocmp"
"google.golang.org/protobuf/types/known/anypb"
trContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/log/logtest"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/source_metadatapb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
@@ -438,7 +443,6 @@ func TestSkipBinaries(t *testing.T) {
source := &Source{
paths: []string{textFile, binaryFile}, // Test individual files
skipBinaries: true,
- log: logr.Discard(),
}
chunks := make(chan *sources.Chunk, 10)
@@ -888,3 +892,92 @@ func createTempDir(dir string, contents ...string) (string, func(), error) {
}
return tmpdir, func() { _ = os.RemoveAll(tmpdir) }, nil
}
+
+func TestSource_DebugLogging(t *testing.T) {
+ const testSecret = "TESTSECRET-do-not-log-1234567890"
+
+ // One file of each kind the source logs about: scanned, binary, excluded by filter, and a symlink.
+ dir := t.TempDir()
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "config.txt"), []byte("password = "+testSecret), 0644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "app.exe"), []byte{0x4D, 0x5A, 0x90, 0x00}, 0644))
+ require.NoError(t, os.WriteFile(filepath.Join(dir, "excluded.txt"), []byte("password = "+testSecret), 0644))
+ require.NoError(t, os.Symlink(filepath.Join(dir, "config.txt"), filepath.Join(dir, "link.txt")))
+
+ excludeFile := filepath.Join(t.TempDir(), "exclude.txt")
+ require.NoError(t, os.WriteFile(excludeFile, []byte("excluded\\.txt$\n"), 0644))
+
+ conn, err := anypb.New(&sourcespb.Filesystem{
+ Paths: []string{dir},
+ SkipBinaries: true,
+ ExcludePathsFile: excludeFile,
+ })
+ require.NoError(t, err)
+
+ // run the source with both Chunks and ChunkUnit, draining chunks the way the engine does
+ runSource := func(ctx trContext.Context) {
+ s := Source{}
+ require.NoError(t, s.Init(ctx, "test debug logging", 1, 1, false, conn, 1))
+
+ chunksCh := make(chan *sources.Chunk, 64)
+ go func() {
+ for range chunksCh {
+ }
+ }()
+ require.NoError(t, s.Chunks(ctx, chunksCh))
+ close(chunksCh)
+
+ reporter := sourcestest.TestReporter{}
+ require.NoError(t, s.ChunkUnit(ctx, sources.CommonSourceUnit{ID: dir}, &reporter))
+ require.Empty(t, reporter.ChunkErrs)
+ }
+
+ t.Run("debug on", func(t *testing.T) {
+ ctx, logs := logtest.DebugContext()
+ ctx = trContext.WithValues(ctx, "source_name", "test debug logging", "job_id", 1)
+ runSource(ctx)
+
+ // lines from each level show up
+ for _, msg := range []string{
+ "filesystem source initialized", // V(1)
+ "scanning path", // V(2)
+ "finished scanning path", // V(2)
+ "scanning directory", // V(3)
+ "scanning file", // V(3)
+ "finished scanning file", // V(5)
+ } {
+ assert.NotZero(t, logs.FilterMessage(msg).Len(), "missing log line %q", msg)
+ }
+
+ // every skip says why, at V(4)
+ var reasons []string
+ for _, entry := range logs.FilterMessage("skipping file").All() {
+ reasons = append(reasons, fmt.Sprint(entry.ContextMap()["reason"]))
+ }
+ for _, entry := range logs.FilterMessage("skipping symlink").All() {
+ reasons = append(reasons, fmt.Sprint(entry.ContextMap()["reason"]))
+ }
+ assert.Contains(t, reasons, "binary files are skipped")
+ assert.Contains(t, reasons, "excluded by path filter")
+ assert.Contains(t, reasons, "following symlinks is not enabled")
+
+ // every line came from the context logger, so it carries the source fields,
+ // and none of them contain the secret
+ for _, entry := range logs.All() {
+ fields := entry.ContextMap()
+ assert.Contains(t, fields, "source_name", "line %q", entry.Message)
+ assert.Contains(t, fields, "job_id", "line %q", entry.Message)
+ assert.NotContains(t, entry.Message, testSecret)
+ assert.NotContains(t, fmt.Sprint(fields), testSecret)
+ }
+ })
+
+ t.Run("debug off", func(t *testing.T) {
+ core, logs := observer.New(zapcore.InfoLevel)
+ ctx := trContext.WithLogger(trContext.Background(), zapr.NewLogger(zap.New(core)))
+ runSource(ctx)
+
+ assert.Zero(t, logs.FilterMessage("scanning file").Len())
+ assert.Zero(t, logs.FilterMessage("skipping file").Len())
+ assert.Zero(t, logs.FilterMessage("finished scanning file").Len())
+ })
+}