Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add optional param to Chunks (#1747)
ahrav2a9f34962d90
diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.goindex 2ae7e6b3b..301f102a1 100644--- a/pkg/sources/circleci/circleci.go+++ b/pkg/sources/circleci/circleci.go@@ -75,7 +75,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver}// Chunks emits chunks of bytes over a channel.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {projects, err := s.projects(ctx)if err != nil {return fmt.Errorf("error getting projects: %w", err)diff --git a/pkg/sources/docker/docker.go b/pkg/sources/docker/docker.goindex 667dbd600..8bd5d48d2 100644--- a/pkg/sources/docker/docker.go+++ b/pkg/sources/docker/docker.go@@ -85,7 +85,7 @@ type layerInfo struct {}// Chunks emits data over a channel that is decoded and scanned for secrets.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {ctx = context.WithValues(ctx, "source_type", s.Type(), "source_name", s.name)workers := new(errgroup.Group)diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex 49f504006..2af668a58 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -77,7 +77,7 @@ func (s *Source) WithFilter(filter *common.Filter) {}// Chunks emits chunks of bytes over a channel.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {for i, path := range s.paths {logger := ctx.Logger().WithValues("path", path)if common.IsDone(ctx) {diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.goindex dfb0a40f3..ab867c3b5 100644--- a/pkg/sources/gcs/gcs.go+++ b/pkg/sources/gcs/gcs.go@@ -248,7 +248,7 @@ func (s *Source) enumerate(ctx context.Context) error {}// Chunks emits chunks of bytes over a channel.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {persistableCache := s.setupCache(ctx)objectCh, err := s.gcsManager.ListObjects(ctx)diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 76301cbae..81560cbdb 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -153,7 +153,7 @@ func (s *Source) Init(aCtx context.Context, name string, jobId, sourceId int64,}// Chunks emits chunks of bytes over a channel.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {if err := s.scanRepos(ctx, chunksChan); err != nil {return err}diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex 3b5d1f016..511a0ec84 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -413,7 +413,7 @@ func (s *Source) visibilityOf(ctx context.Context, repoURL string) (visibility s}// Chunks emits chunks of bytes over a channel.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {apiEndpoint := s.conn.Endpointif len(apiEndpoint) == 0 || endsWithGithub.MatchString(apiEndpoint) {apiEndpoint = "https://api.github.com"diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 9cfacea4d..06c5f5d00 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -139,7 +139,7 @@ func (s *Source) Init(_ context.Context, name string, jobId, sourceId int64, ver}// Chunks emits chunks of bytes over a channel.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {// Start client.apiClient, err := s.newClient()if err != nil {diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.goindex 07b1a19dd..a9f80732e 100644--- a/pkg/sources/s3/s3.go+++ b/pkg/sources/s3/s3.go@@ -232,7 +232,7 @@ func (s *Source) scanBuckets(ctx context.Context, client *s3.S3, role string, bu}// Chunks emits chunks of bytes over a channel.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {visitor := func(c context.Context, defaultRegionClient *s3.S3, roleArn string, buckets []string) {s.scanBuckets(c, defaultRegionClient, roleArn, buckets, chunksChan)}diff --git a/pkg/sources/source_manager_test.go b/pkg/sources/source_manager_test.goindex 9a196a897..31378eb02 100644--- a/pkg/sources/source_manager_test.go+++ b/pkg/sources/source_manager_test.go@@ -6,10 +6,11 @@ import ("testing""github.com/stretchr/testify/assert"+ "google.golang.org/protobuf/types/known/anypb"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/sourcespb"- "google.golang.org/protobuf/types/known/anypb")// DummySource implements Source and is used for testing a SourceManager.@@ -31,7 +32,7 @@ func (d *DummySource) GetProgress() *Progress { return nil }// Interface to easily test different chunking methods.type chunker interface {- Chunks(context.Context, chan *Chunk) error+ Chunks(context.Context, chan *Chunk, ...ChunkingTarget) errorChunkUnit(ctx context.Context, unit SourceUnit, reporter ChunkReporter) errorEnumerate(ctx context.Context, reporter UnitReporter) error}@@ -42,7 +43,7 @@ type counterChunker struct {count int}-func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk) error {+func (c *counterChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {for i := 0; i < c.count; i++ {select {case ch <- &Chunk{Data: []byte{c.chunkCounter}}:@@ -75,9 +76,9 @@ func (c *counterChunker) ChunkUnit(ctx context.Context, unit SourceUnit, reporte// Chunk method that always returns an error.type errorChunker struct{ error }-func (c errorChunker) Chunks(context.Context, chan *Chunk) error { return c }-func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }-func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }+func (c errorChunker) Chunks(context.Context, chan *Chunk, ...ChunkingTarget) error { return c }+func (c errorChunker) Enumerate(context.Context, UnitReporter) error { return c }+func (c errorChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return c }// enrollDummy is a helper function to enroll a DummySource with a SourceManager.func enrollDummy(mgr *SourceManager, chunkMethod chunker) (handle, error) {@@ -176,7 +177,7 @@ type unitChunk struct {type unitChunker struct{ steps []unitChunk }-func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk) error {+func (c *unitChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {for _, step := range c.steps {if step.err != "" {continue@@ -294,7 +295,9 @@ type callbackChunker struct {cb func(context.Context, chan *Chunk) error}-func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk) error { return c.cb(ctx, ch) }+func (c callbackChunker) Chunks(ctx context.Context, ch chan *Chunk, _ ...ChunkingTarget) error {+ return c.cb(ctx, ch)+}func (c callbackChunker) Enumerate(context.Context, UnitReporter) error { return nil }func (c callbackChunker) ChunkUnit(context.Context, SourceUnit, ChunkReporter) error { return nil }diff --git a/pkg/sources/sources.go b/pkg/sources/sources.goindex 07da7ccaf..eba823a6c 100644--- a/pkg/sources/sources.go+++ b/pkg/sources/sources.go@@ -30,6 +30,16 @@ type Chunk struct {Verify bool}+// ChunkingTarget specifies criteria for a targeted chunking process.+// Instead of collecting data indiscriminately, this struct allows the caller+// to specify particular subsets of data they're interested in. This becomes+// especially useful when one needs to verify or recheck specific data points+// without processing the entire dataset.+type ChunkingTarget struct {+ // QueryCriteria represents specific parameters or conditions to target the chunking process.+ QueryCriteria source_metadatapb.MetaData+}+// Source defines the interface required to implement a source chunker.type Source interface {// Type returns the source type, used for matching against configuration and jobs.@@ -40,8 +50,12 @@ type Source interface {JobID() int64// Init initializes the source.Init(aCtx context.Context, name string, jobId, sourceId int64, verify bool, connection *anypb.Any, concurrency int) error- // Chunks emits data over a channel that is decoded and scanned for secrets.- Chunks(ctx context.Context, chunksChan chan *Chunk) error+ // Chunks emits data over a channel which is then decoded and scanned for secrets.+ // By default, data is obtained indiscriminately. However, by providing one or more+ // ChunkingTarget parameters, the caller can direct the function to retrieve+ // specific chunks of data. This targeted approach allows for efficient and+ // intentional data processing, beneficial when verifying or rechecking specific data points.+ Chunks(ctx context.Context, chunksChan chan *Chunk, targets ...ChunkingTarget) error// GetProgress is the completion progress (percentage) for Scanned Source.GetProgress() *Progress}diff --git a/pkg/sources/syslog/syslog.go b/pkg/sources/syslog/syslog.goindex d9be9d117..76814b03e 100644--- a/pkg/sources/syslog/syslog.go+++ b/pkg/sources/syslog/syslog.go@@ -182,7 +182,7 @@ func (s *Source) verifyConnectionConfig() error {}// Chunks emits chunks of bytes over a channel.-func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk) error {+func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ ...sources.ChunkingTarget) error {switch {case s.conn.TlsCert != nilString || s.conn.TlsKey != nilString:cert, err := tls.X509KeyPair([]byte(s.conn.TlsCert), []byte(s.conn.TlsKey))