Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Use the common chunker for scanning the filesystem source (#1619)
ahrave8945406321f
diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex 41673a4e2..89d473486 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -1,7 +1,6 @@package filesystemimport (- "bufio""fmt""io""io/fs"@@ -23,13 +22,6 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/sources")-const (- // These buffer sizes are mainly driven by our largest credential size, which is GCP @ ~2.25KB.- // Having a peek size larger than that ensures that we have complete credential coverage in our chunks.- BufferSize = 10 * 1024 // 10KB- PeekSize = 3 * 1024 // 3KB-)-type Source struct {name stringsourceId int64@@ -187,34 +179,33 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou}reReader.Stop()- for {- chunkBytes := make([]byte, BufferSize)- reader := bufio.NewReaderSize(reReader, BufferSize)- n, err := reader.Read(chunkBytes)- if err != nil && !errors.Is(err, io.EOF) {- break+ chunkReader := sources.NewChunkReader()+ chunkResChan := chunkReader(ctx, reReader)+ for data := range chunkResChan {+ if err := data.Error(); err != nil {+ s.log.Error(err, "error reading chunk.")+ continue}- peekData, _ := reader.Peek(PeekSize)- if n > 0 {- chunksChan <- &sources.Chunk{- SourceType: s.Type(),- SourceName: s.name,- SourceID: s.SourceID(),- Data: append(chunkBytes[:n], peekData...),- SourceMetadata: &source_metadatapb.MetaData{- Data: &source_metadatapb.MetaData_Filesystem{- Filesystem: &source_metadatapb.Filesystem{- File: sanitizer.UTF8(path),- },++ chunk := &sources.Chunk{+ SourceType: s.Type(),+ SourceName: s.name,+ SourceID: s.SourceID(),+ Data: data.Bytes(),+ SourceMetadata: &source_metadatapb.MetaData{+ Data: &source_metadatapb.MetaData_Filesystem{+ Filesystem: &source_metadatapb.Filesystem{+ File: sanitizer.UTF8(path),},},- Verify: s.verify,- }+ },+ Verify: s.verify,}- if errors.Is(err, io.EOF) {- break+ if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {+ return err}}+return nil}diff --git a/pkg/sources/filesystem/filesystem_test.go b/pkg/sources/filesystem/filesystem_test.goindex 7d56bc9d9..ef5dfbdd3 100644--- a/pkg/sources/filesystem/filesystem_test.go+++ b/pkg/sources/filesystem/filesystem_test.go@@ -1,10 +1,14 @@package filesystemimport (+ "os"+ "strings""testing""time"+ "github.com/go-logr/logr""github.com/kylelemons/godebug/pretty"+ "github.com/stretchr/testify/assert""google.golang.org/protobuf/types/known/anypb""github.com/trufflesecurity/trufflehog/v3/pkg/context"@@ -78,3 +82,41 @@ func TestSource_Scan(t *testing.T) {})}}++func TestScanFile(t *testing.T) {+ tmpfile, err := os.CreateTemp("", "example.txt")+ if err != nil {+ t.Fatal(err)+ }+ defer os.Remove(tmpfile.Name())++ chunkSize := sources.ChunkSize+ secretPart1 := "SECRET"+ secretPart2 := "SPLIT"+ // Split the secret into two parts and pad the rest of the chunk with A's.+ data := strings.Repeat("A", chunkSize-len(secretPart1)) + secretPart1 + secretPart2 + strings.Repeat("A", chunkSize-len(secretPart2))++ _, err = tmpfile.Write([]byte(data))+ assert.Nil(t, err)++ err = tmpfile.Close()+ assert.Nil(t, err)++ source := &Source{}+ chunksChan := make(chan *sources.Chunk, 2)++ ctx := context.WithLogger(context.Background(), logr.Discard())+ go func() {+ defer close(chunksChan)+ err = source.scanFile(ctx, tmpfile.Name(), chunksChan)+ assert.Nil(t, err)+ }()++ // Read from the channel and validate the secrets.+ foundSecret := ""+ for chunkCh := range chunksChan {+ foundSecret += string(chunkCh.Data)+ }++ assert.Contains(t, foundSecret, secretPart1+secretPart2)+}