Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Move verify flag into `detectableChunk` (#4558)
Cody Rose041f07e9df90
diff --git a/pkg/decoders/escaped_unicode.go b/pkg/decoders/escaped_unicode.goindex f3f6462b3..b8574bc10 100644--- a/pkg/decoders/escaped_unicode.go+++ b/pkg/decoders/escaped_unicode.go@@ -111,7 +111,7 @@ func (d *EscapedUnicode) FromChunk(chunk *sources.Chunk) *DecodableChunk {SecretID: chunk.SecretID,SourceMetadata: chunk.SourceMetadata,SourceType: chunk.SourceType,- Verify: chunk.Verify,+ SourceVerify: chunk.SourceVerify,},}} else {diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex e45651dca..5e64ba63e 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -777,6 +777,7 @@ type detectableChunk struct {chunk sources.Chunkdecoder detectorspb.DecoderTypewgDoneFn func()+ verify bool}// verificationOverlapChunk is a decoded chunk that has multiple detectors that match it.@@ -851,7 +852,7 @@ func (e *Engine) scannerWorker(ctx context.Context) {for chunk := range e.ChunksChan() {startTime := time.Now()- sourceVerify := chunk.Verify+ sourceVerify := chunk.SourceVerifydecoded := iterativeDecode(chunk, e.decoders, e.maxDecodeDepth)@@ -869,12 +870,12 @@ func (e *Engine) scannerWorker(ctx context.Context) {}for _, detector := range matchingDetectors {- d.Chunk.Verify = e.shouldVerifyChunk(sourceVerify, detector, e.detectorVerificationOverrides)wgDetect.Add(1)e.detectableChunksChan <- detectableChunk{chunk: *d.Chunk,detector: detector,decoder: d.DecoderType,+ verify: e.shouldVerifyChunk(sourceVerify, detector, e.detectorVerificationOverrides),wgDoneFn: wgDetect.Done,}}@@ -1069,11 +1070,11 @@ func (e *Engine) verificationOverlapWorker(ctx context.Context) {for _, detector := range detectorKeysWithResults {wgDetect.Add(1)- chunk.chunk.Verify = e.shouldVerifyChunk(chunk.chunk.Verify, detector, e.detectorVerificationOverrides)e.detectableChunksChan <- detectableChunk{chunk: chunk.chunk,detector: detector,decoder: chunk.decoder,+ verify: e.shouldVerifyChunk(chunk.chunk.SourceVerify, detector, e.detectorVerificationOverrides),wgDoneFn: wgDetect.Done,}}@@ -1136,7 +1137,7 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {results, err := e.verificationCache.FromData(ctx,data.detector.Detector,- data.chunk.Verify,+ data.verify,data.chunk.SecretID != 0,matchBytes)t.Stop()diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.goindex ec4df8603..27fa6ceb3 100644--- a/pkg/engine/engine_test.go+++ b/pkg/engine/engine_test.go@@ -1501,77 +1501,115 @@ func (p passthroughDecoder) FromChunk(chunk *sources.Chunk) *decoders.DecodableCfunc (p passthroughDecoder) Type() detectorspb.DecoderType { return detectorspb.DecoderType(-1) }+// TestEngine_DetectChunk_UsesVerifyFlag validates that detectChunk correctly forwards detectableChunk.verify to+// detectors.func TestEngine_DetectChunk_UsesVerifyFlag(t *testing.T) {ctx := context.Background()- // Arrange: Create a minimal engine.- e := &Engine{- results: make(chan detectors.ResultWithMetadata, 1),- verificationCache: verificationcache.New(nil, &verificationcache.InMemoryMetrics{}),+ testCases := []struct {+ name string+ verify bool+ }{+ {name: "verify=true", verify: true},+ {name: "verify=false", verify: false},}- // Arrange: Create a detector match. We can't create one directly, so we have to use a minimal A-H core.- ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{passthroughDetector{keywords: []string{"keyword"}}})- detectorMatches := ahcore.FindDetectorMatches([]byte("keyword"))- require.Len(t, detectorMatches, 1)+ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ // Arrange: Create a minimal engine.+ e := &Engine{+ results: make(chan detectors.ResultWithMetadata, 1),+ verificationCache: verificationcache.New(nil, &verificationcache.InMemoryMetrics{}),+ }- // Arrange: Create a chunk to detect.- chunk := detectableChunk{- chunk: sources.Chunk{- Verify: true,- },- detector: detectorMatches[0],- wgDoneFn: func() {},- }+ // Arrange: Create a detector match. We can't create one directly, so we have to use a minimal A-H core.+ ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{passthroughDetector{keywords: []string{"keyword"}}})+ detectorMatches := ahcore.FindDetectorMatches([]byte("keyword"))+ require.Len(t, detectorMatches, 1)- // Act- e.detectChunk(ctx, chunk)- close(e.results)+ // Arrange: Create a chunk to detect.+ chunk := detectableChunk{+ detector: detectorMatches[0],+ verify: tc.verify,+ wgDoneFn: func() {},+ }- // Assert: Confirm that a result was generated and that it has the expected verify flag.- select {- case result := <-e.results:- assert.True(t, result.Result.Verified)- default:- t.Errorf("expected a result but did not get one")+ // Act+ e.detectChunk(ctx, chunk)+ close(e.results)++ // Assert: Confirm that a result was generated and that it has the expected verify flag.+ select {+ case result := <-e.results:+ assert.Equal(t, tc.verify, result.Result.Verified)+ default:+ t.Errorf("expected a result but did not get one")+ }+ })}}+// TestEngine_ScannerWorker_DetectableChunkHasCorrectVerifyFlag validates that scannerWorker generates detectableChunk+// structs that have the correct verify flag set. It also validates that the original chunks' SourceVerify flags are+// unchanged.func TestEngine_ScannerWorker_DetectableChunkHasCorrectVerifyFlag(t *testing.T) {ctx := context.Background()- // Arrange: Create a minimal engine.- detector := &passthroughDetector{keywords: []string{"keyword"}}- e := &Engine{- AhoCorasickCore: ahocorasick.NewAhoCorasickCore([]detectors.Detector{detector}),- decoders: []decoders.Decoder{passthroughDecoder{}},- detectableChunksChan: make(chan detectableChunk, 1),- sourceManager: sources.NewManager(),- verify: true,- maxDecodeDepth: 1,+ testCases := []struct {+ name string+ engineVerify bool+ sourceVerify bool+ wantVerify bool+ }{+ {name: "engineVerify=false,sourceVerify=false", engineVerify: false, sourceVerify: false, wantVerify: false},+ {name: "engineVerify=false,sourceVerify=true", engineVerify: false, sourceVerify: true, wantVerify: false},+ {name: "engineVerify=true,sourceVerify=false", engineVerify: true, sourceVerify: false, wantVerify: false},+ {name: "engineVerify=true,sourceVerify=true", engineVerify: true, sourceVerify: true, wantVerify: true},}- // Arrange: Create a chunk to scan.- chunk := sources.Chunk{- Data: []byte("keyword"),- Verify: true,- }+ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ // Arrange: Create a minimal engine.+ detector := &passthroughDetector{keywords: []string{"keyword"}}+ e := &Engine{+ AhoCorasickCore: ahocorasick.NewAhoCorasickCore([]detectors.Detector{detector}),+ decoders: []decoders.Decoder{passthroughDecoder{}},+ detectableChunksChan: make(chan detectableChunk, 1),+ sourceManager: sources.NewManager(),+ verify: tc.engineVerify,+ maxDecodeDepth: 1,+ }- // Arrange: Enqueue a chunk to be scanned.- e.sourceManager.ScanChunk(&chunk)+ // Arrange: Create a chunk to scan.+ chunk := sources.Chunk{+ Data: []byte("keyword"),+ SourceVerify: tc.sourceVerify,+ }- // Act- go e.scannerWorker(ctx)+ // Arrange: Enqueue a chunk to be scanned.+ e.sourceManager.ScanChunk(&chunk)- // Assert: Confirm that a chunk was generated and that it has the expected verify flag.- select {- case chunk := <-e.detectableChunksChan:- assert.True(t, chunk.chunk.Verify)- case <-time.After(1 * time.Second):- t.Errorf("expected a detectableChunk but did not get one")+ // Act+ go e.scannerWorker(ctx)++ // Assert: Confirm that a chunk was generated, that its SourceVerify flag is unchanged, and that its verify+ // flag is correctly set.+ select {+ case chunk := <-e.detectableChunksChan:+ assert.Equal(t, tc.sourceVerify, chunk.chunk.SourceVerify)+ assert.Equal(t, tc.wantVerify, chunk.verify)+ case <-time.After(1 * time.Second):+ t.Errorf("expected a detectableChunk but did not get one")+ }+ })}}+// TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag validates that the results directly+// generated by verificationOverlapWorker all came from detector invocations with the verify flag cleared (because these+// results were generated from verification overlaps). It also validates that detectableChunk structs generated by+// verificationOverlapWorker have their verify flags correctly set, and that these structs' original chunks'+// SourceVerify flags are unchanged.func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(t *testing.T) {ctx := context.Background()@@ -1597,8 +1635,8 @@ func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(t// Arrange: Create a chunk to "scan."chunk := sources.Chunk{- Data: []byte("keyword ;oahpow8heg;blaisd"),- Verify: true,+ Data: []byte("keyword ;oahpow8heg;blaisd"),+ SourceVerify: true,}// Arrange: Create overlapping detector matches. We can't create them directly, so we have to use a minimal A-H@@ -1628,11 +1666,13 @@ func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(tassert.False(t, result.Result.Verified)}- // Assert: Confirm that every generated detectable chunk carries the original Verify flag.+ // Assert: Confirm that every generated detectable chunk's Chunk.SourceVerify flag is unchanged and that its+ // verify flag is correctly set.// CMR: There should be not be any of these chunks. However, due to what I believe is an unrelated bug, there- // are. This test ensures that even in that erroneous case, their Verify flag is correct.+ // are. This test ensures that even in that erroneous case, their contents are correct.for detectableChunk := range processedDetectableChunks {- assert.True(t, detectableChunk.chunk.Verify)+ assert.True(t, detectableChunk.verify)+ assert.True(t, detectableChunk.chunk.SourceVerify)}})t.Run("no overlap", func(t *testing.T) {@@ -1656,8 +1696,8 @@ func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(t// Arrange: Create a chunk to "scan."chunk := sources.Chunk{- Data: []byte("keyword ;oahpow8heg;blaisd"),- Verify: true,+ Data: []byte("keyword ;oahpow8heg;blaisd"),+ SourceVerify: true,}// Arrange: Create non-overlapping detector matches. We can't create them directly, so we have to use a minimal@@ -1681,9 +1721,10 @@ func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(tclose(e.detectableChunksChan)close(processedDetectableChunks)- // Assert: Confirm that every generated detectable chunk carries the original Verify flag.+ // Assert: Confirm that SourceVerify flags are unchanged, and verify flags are correctly set.for detectableChunk := range processedDetectableChunks {- assert.True(t, detectableChunk.chunk.Verify)+ assert.True(t, detectableChunk.chunk.SourceVerify)+ assert.True(t, detectableChunk.verify)}})}diff --git a/pkg/sources/circleci/circleci.go b/pkg/sources/circleci/circleci.goindex 667974b88..5dee11109 100644--- a/pkg/sources/circleci/circleci.go+++ b/pkg/sources/circleci/circleci.go@@ -375,7 +375,7 @@ func (s *Source) chunk(ctx context.Context, proj project, buildNum BuildNum, ste},},},- Verify: s.verify,+ SourceVerify: s.verify,}if err := data.Error(); err != nil {return errdiff --git a/pkg/sources/docker/docker.go b/pkg/sources/docker/docker.goindex 53aa60665..135bb10c4 100644--- a/pkg/sources/docker/docker.go+++ b/pkg/sources/docker/docker.go@@ -351,8 +351,8 @@ func (s *Source) processHistoryEntry(ctx context.Context, historyInfo historyEnt},},},- Verify: s.verify,- Data: []byte(historyInfo.entry.CreatedBy),+ SourceVerify: s.verify,+ Data: []byte(historyInfo.entry.CreatedBy),}ctx.Logger().V(2).Info("scanning image history entry", "index", historyInfo.index, "layer", historyInfo.layerDigest)@@ -464,7 +464,7 @@ func (s *Source) processChunk(ctx context.Context, info chunkProcessingInfo, chu},},},- Verify: s.verify,+ SourceVerify: s.verify,}chunk.Data = data.Bytes()diff --git a/pkg/sources/elasticsearch/elasticsearch.go b/pkg/sources/elasticsearch/elasticsearch.goindex 2d8226e4c..a7bdc95f8 100644--- a/pkg/sources/elasticsearch/elasticsearch.go+++ b/pkg/sources/elasticsearch/elasticsearch.go@@ -202,7 +202,7 @@ func (s *Source) Chunks(},},},- Verify: s.verify,+ SourceVerify: s.verify,}chunk.Data = []byte(document.message)diff --git a/pkg/sources/filesystem/filesystem.go b/pkg/sources/filesystem/filesystem.goindex faf36ee3d..e93ac6154 100644--- a/pkg/sources/filesystem/filesystem.go+++ b/pkg/sources/filesystem/filesystem.go@@ -346,7 +346,7 @@ func (s *Source) scanFile(ctx context.Context, path string, chunksChan chan *sou},},},- Verify: s.verify,+ SourceVerify: s.verify,}return handlers.HandleFile(fileCtx, inputFile, chunkSkel, sources.ChanReporter{Ch: chunksChan})diff --git a/pkg/sources/gcs/gcs.go b/pkg/sources/gcs/gcs.goindex f23c4f9ee..413dde690 100644--- a/pkg/sources/gcs/gcs.go+++ b/pkg/sources/gcs/gcs.go@@ -333,11 +333,11 @@ func (s *Source) completeProgress(ctx context.Context) {func (s *Source) processObject(ctx context.Context, o object) error {chunkSkel := &sources.Chunk{- SourceName: s.name,- SourceType: s.Type(),- JobID: s.JobID(),- SourceID: s.sourceId,- Verify: s.verify,+ SourceName: s.name,+ SourceType: s.Type(),+ JobID: s.JobID(),+ SourceID: s.sourceId,+ SourceVerify: s.verify,SourceMetadata: &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Gcs{Gcs: &source_metadatapb.GCS{diff --git a/pkg/sources/gcs/gcs_integration_test.go b/pkg/sources/gcs/gcs_integration_test.goindex 6c8fe2a09..3fa3a8865 100644--- a/pkg/sources/gcs/gcs_integration_test.go+++ b/pkg/sources/gcs/gcs_integration_test.go@@ -79,10 +79,10 @@ func TestChunks_PublicBucket(t *testing.T) {want := []*sources.Chunk{{- SourceName: "test",- SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,- SourceID: 0,- Verify: true,+ SourceName: "test",+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,+ SourceID: 0,+ SourceVerify: true,SourceMetadata: &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Gcs{Gcs: &source_metadatapb.GCS{@@ -164,10 +164,10 @@ func createTestChunks() []*sources.Chunk {chunks := make([]*sources.Chunk, 0, len(objects))for _, o := range objects {chunks = append(chunks, &sources.Chunk{- SourceName: "test",- SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,- SourceID: 0,- Verify: true,+ SourceName: "test",+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,+ SourceID: 0,+ SourceVerify: true,SourceMetadata: &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Gcs{Gcs: &source_metadatapb.GCS{diff --git a/pkg/sources/gcs/gcs_test.go b/pkg/sources/gcs/gcs_test.goindex 126facfe4..9cde72ffe 100644--- a/pkg/sources/gcs/gcs_test.go+++ b/pkg/sources/gcs/gcs_test.go@@ -271,11 +271,11 @@ func createTestObject(id int) object {func createTestSourceChunk(id int) *sources.Chunk {return &sources.Chunk{- SourceName: "test",- SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,- SourceID: 0,- Verify: true,- Data: []byte(fmt.Sprintf("hello world %d", id)),+ SourceName: "test",+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GCS,+ SourceID: 0,+ SourceVerify: true,+ Data: []byte(fmt.Sprintf("hello world %d", id)),SourceMetadata: &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Gcs{Gcs: &source_metadatapb.GCS{diff --git a/pkg/sources/git/git.go b/pkg/sources/git/git.goindex 2aada8b02..59696b6e9 100644--- a/pkg/sources/git/git.go+++ b/pkg/sources/git/git.go@@ -762,7 +762,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringSourceType: s.sourceType,SourceMetadata: metadata,Data: []byte(sb.String()),- Verify: s.verify,+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {return err@@ -796,7 +796,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringJobID: s.jobID,SourceType: s.sourceType,SourceMetadata: metadata,- Verify: s.verify,+ SourceVerify: s.verify,}if err := HandleBinary(ctx, gitDir, reporter, chunkSkel, commitHash, fileName, s.skipArchives); err != nil {@@ -845,7 +845,7 @@ func (s *Git) ScanCommits(ctx context.Context, repo *git.Repository, path stringSourceType: s.sourceType,SourceMetadata: metadata,Data: data,- Verify: s.verify,+ SourceVerify: s.verify,}return reporter.ChunkOk(ctx, chunk)}@@ -883,7 +883,7 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, emailSourceType: s.sourceType,SourceMetadata: metadata,Data: append([]byte{}, newChunkBuffer.Bytes()...),- Verify: s.verify,+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {// TODO: Return error.@@ -903,7 +903,7 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, emailSourceType: s.sourceType,SourceMetadata: metadata,Data: line,- Verify: s.verify,+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {// TODO: Return error.@@ -927,7 +927,7 @@ func (s *Git) gitChunk(ctx context.Context, diff *gitparse.Diff, fileName, emailSourceType: s.sourceType,SourceMetadata: metadata,Data: append([]byte{}, newChunkBuffer.Bytes()...),- Verify: s.verify,+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {// TODO: Return error.@@ -1027,7 +1027,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,JobID: s.jobID,SourceType: s.sourceType,SourceMetadata: metadata,- Verify: s.verify,+ SourceVerify: s.verify,}if err := HandleBinary(ctx, gitDir, reporter, chunkSkel, commitHash, fileName, s.skipArchives); err != nil {logger.Error(err, "error handling binary file")@@ -1057,7 +1057,7 @@ func (s *Git) ScanStaged(ctx context.Context, repo *git.Repository, path string,SourceType: s.sourceType,SourceMetadata: metadata,Data: data,- Verify: s.verify,+ SourceVerify: s.verify,}return reporter.ChunkOk(ctx, chunk)}diff --git a/pkg/sources/git/git_test.go b/pkg/sources/git/git_test.goindex 744218b83..f419eee5e 100644--- a/pkg/sources/git/git_test.go+++ b/pkg/sources/git/git_test.go@@ -96,9 +96,9 @@ func TestSource_Scan(t *testing.T) {concurrency: 4,},wantChunk: &sources.Chunk{- SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,- SourceName: "this repo",- Verify: false,+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,+ SourceName: "this repo",+ SourceVerify: false,},wantErr: false,},@@ -115,9 +115,9 @@ func TestSource_Scan(t *testing.T) {concurrency: 4,},wantChunk: &sources.Chunk{- SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,- SourceName: "test source",- Verify: false,+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,+ SourceName: "test source",+ SourceVerify: false,},wantErr: false,},@@ -134,9 +134,9 @@ func TestSource_Scan(t *testing.T) {concurrency: 0,},wantChunk: &sources.Chunk{- SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,- SourceName: "test source",- Verify: false,+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,+ SourceName: "test source",+ SourceVerify: false,},wantErr: false,},@@ -156,9 +156,9 @@ func TestSource_Scan(t *testing.T) {concurrency: 4,},wantChunk: &sources.Chunk{- SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,- SourceName: "test source",- Verify: false,+ SourceType: sourcespb.SourceType_SOURCE_TYPE_GIT,+ SourceName: "test source",+ SourceVerify: false,},wantErr: false,},diff --git a/pkg/sources/github/github.go b/pkg/sources/github/github.goindex aa1112d29..28f129890 100644--- a/pkg/sources/github/github.go+++ b/pkg/sources/github/github.go@@ -1252,8 +1252,8 @@ func (s *Source) chunkGistComments(ctx context.Context, gistURL string, gistInfo},},},- Data: []byte(sanitizer.UTF8(comment.GetBody())),- Verify: s.verify,+ Data: []byte(sanitizer.UTF8(comment.GetBody())),+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {@@ -1389,8 +1389,8 @@ func (s *Source) chunkIssues(ctx context.Context, repoInfo repoInfo, issues []*g},},},- Data: []byte(sanitizer.UTF8(issue.GetTitle() + "\n" + issue.GetBody())),- Verify: s.verify,+ Data: []byte(sanitizer.UTF8(issue.GetTitle() + "\n" + issue.GetBody())),+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {@@ -1456,8 +1456,8 @@ func (s *Source) chunkIssueComments(ctx context.Context, repoInfo repoInfo, comm},},},- Data: []byte(sanitizer.UTF8(comment.GetBody())),- Verify: s.verify,+ Data: []byte(sanitizer.UTF8(comment.GetBody())),+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {@@ -1552,8 +1552,8 @@ func (s *Source) chunkPullRequests(ctx context.Context, repoInfo repoInfo, prs [},},},- Data: []byte(sanitizer.UTF8(pr.GetTitle() + "\n" + pr.GetBody())),- Verify: s.verify,+ Data: []byte(sanitizer.UTF8(pr.GetTitle() + "\n" + pr.GetBody())),+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {@@ -1588,8 +1588,8 @@ func (s *Source) chunkPullRequestComments(ctx context.Context, repoInfo repoInfo},},},- Data: []byte(sanitizer.UTF8(comment.GetBody())),- Verify: s.verify,+ Data: []byte(sanitizer.UTF8(comment.GetBody())),+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {@@ -1627,7 +1627,7 @@ func (s *Source) scanTarget(ctx context.Context, target sources.ChunkingTarget,SourceMetadata: &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Github{Github: meta},},- Verify: s.verify,+ SourceVerify: s.verify,}u, err := url.Parse(meta.GetLink())diff --git a/pkg/sources/github/github_integration_test.go b/pkg/sources/github/github_integration_test.goindex 6e5ce443b..1bbed6305 100644--- a/pkg/sources/github/github_integration_test.go+++ b/pkg/sources/github/github_integration_test.go@@ -127,7 +127,7 @@ func TestSource_ScanComments(t *testing.T) {},},},- Verify: false,+ SourceVerify: false,},wantErr: false,},@@ -157,7 +157,7 @@ func TestSource_ScanComments(t *testing.T) {},},},- Verify: false,+ SourceVerify: false,},wantErr: false,},@@ -342,7 +342,7 @@ func TestSource_Scan(t *testing.T) {},},},- Verify: false,+ SourceVerify: false,},wantErr: false,},@@ -367,7 +367,7 @@ func TestSource_Scan(t *testing.T) {},},},- Verify: false,+ SourceVerify: false,},wantErr: false,},@@ -453,7 +453,7 @@ func TestSource_Scan(t *testing.T) {},},},- Verify: false,+ SourceVerify: false,},wantErr: false,},@@ -502,7 +502,7 @@ func TestSource_Scan(t *testing.T) {},},},- Verify: false,+ SourceVerify: false,},wantErr: false,minRepo: 1,@@ -596,7 +596,7 @@ func TestSource_paginateGists(t *testing.T) {},},},- Verify: false,+ SourceVerify: false,},wantErr: false,user: "truffle-sandbox",@@ -994,7 +994,7 @@ func TestSource_ScanCommentsWithGraphql(t *testing.T) {},},},- Verify: false,+ SourceVerify: false,}s := Source{}diff --git a/pkg/sources/github/graphql.go b/pkg/sources/github/graphql.goindex e7e2f08cf..b82a93fe1 100644--- a/pkg/sources/github/graphql.go+++ b/pkg/sources/github/graphql.go@@ -418,8 +418,8 @@ func (s *Source) chunkGraphqlIssues(ctx context.Context, repoInfo repoInfo, issu},},},- Data: []byte(sanitizer.UTF8(issue.Title + "\n" + issue.Body)),- Verify: s.verify,+ Data: []byte(sanitizer.UTF8(issue.Title + "\n" + issue.Body)),+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {@@ -453,8 +453,8 @@ func (s *Source) chunkComments(ctx context.Context, repoInfo repoInfo, comments},},},- Data: []byte(sanitizer.UTF8(comment.Body)),- Verify: s.verify,+ Data: []byte(sanitizer.UTF8(comment.Body)),+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {@@ -483,8 +483,8 @@ func (s *Source) chunkGraphqlPullRequests(ctx context.Context, repoInfo repoInfo},},},- Data: []byte(sanitizer.UTF8(pr.Title + "\n" + pr.Body)),- Verify: s.verify,+ Data: []byte(sanitizer.UTF8(pr.Title + "\n" + pr.Body)),+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {diff --git a/pkg/sources/gitlab/gitlab.go b/pkg/sources/gitlab/gitlab.goindex 5aefa98a1..ec4568aea 100644--- a/pkg/sources/gitlab/gitlab.go+++ b/pkg/sources/gitlab/gitlab.go@@ -388,7 +388,7 @@ func (s *Source) scanTarget(ctx context.Context, client *gitlab.Client, target sSourceMetadata: &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Gitlab{Gitlab: meta},},- Verify: s.verify,+ SourceVerify: s.verify,}if err := common.CancellableWrite(ctx, chunksChan, chunk); err != nil {diff --git a/pkg/sources/huggingface/huggingface.go b/pkg/sources/huggingface/huggingface.goindex f64688aba..80c3733ee 100644--- a/pkg/sources/huggingface/huggingface.go+++ b/pkg/sources/huggingface/huggingface.go@@ -663,8 +663,8 @@ func (s *Source) chunkDiscussionComments(ctx context.Context, repoInfo repoInfo,},},},- Data: []byte(comment.Data.Latest.Raw),- Verify: s.verify,+ Data: []byte(comment.Data.Latest.Raw),+ SourceVerify: s.verify,}select {case <-ctx.Done():diff --git a/pkg/sources/jenkins/jenkins.go b/pkg/sources/jenkins/jenkins.goindex 88a4b627d..e51805555 100644--- a/pkg/sources/jenkins/jenkins.go+++ b/pkg/sources/jenkins/jenkins.go@@ -426,7 +426,7 @@ func (s *Source) chunkBuild(},},},- Verify: s.verify,+ SourceVerify: s.verify,}ctx.Logger().V(4).Info("scanning build log")diff --git a/pkg/sources/json_enumerator/json_enumerator.go b/pkg/sources/json_enumerator/json_enumerator.goindex de86b8ab8..a82cea96d 100644--- a/pkg/sources/json_enumerator/json_enumerator.go+++ b/pkg/sources/json_enumerator/json_enumerator.go@@ -165,11 +165,11 @@ func (s *Source) chunkJSONEnumeratorReader(}chunkSkel := &sources.Chunk{- SourceType: s.Type(),- SourceName: s.name,- SourceID: s.SourceID(),- JobID: s.JobID(),- Verify: s.verify,+ SourceType: s.Type(),+ SourceName: s.name,+ SourceID: s.SourceID(),+ JobID: s.JobID(),+ SourceVerify: s.verify,SourceMetadata: &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_JsonEnumerator{JsonEnumerator: &source_metadatapb.JSONEnumerator{diff --git a/pkg/sources/postman/postman.go b/pkg/sources/postman/postman.goindex da904946a..b6e6140af 100644--- a/pkg/sources/postman/postman.go+++ b/pkg/sources/postman/postman.go@@ -804,7 +804,7 @@ func (s *Source) scanData(ctx context.Context, chunksChan chan *sources.Chunk, d},},},- Verify: s.verify,+ SourceVerify: s.verify,}}diff --git a/pkg/sources/s3/s3.go b/pkg/sources/s3/s3.goindex 2c19d939b..8b0939f20 100644--- a/pkg/sources/s3/s3.go+++ b/pkg/sources/s3/s3.go@@ -596,7 +596,7 @@ func (s *Source) pageChunker(},},},- Verify: s.verify,+ SourceVerify: s.verify,}if err := handlers.HandleFile(ctx, res.Body, chunkSkel, reporter); err != nil {diff --git a/pkg/sources/sources.go b/pkg/sources/sources.goindex d5b45e227..761b7b0ac 100644--- a/pkg/sources/sources.go+++ b/pkg/sources/sources.go@@ -45,8 +45,8 @@ type Chunk struct {// SourceType is the type of Source that produced the chunk.SourceType sourcespb.SourceType- // Verify specifies whether any secrets in the Chunk should be verified.- Verify bool+ // SourceVerify specifies whether this chunk was generated by a source that has verification enabled in its config.+ SourceVerify bool}// ChunkingTarget specifies criteria for a targeted chunking process.diff --git a/pkg/sources/stdin/stdin.go b/pkg/sources/stdin/stdin.goindex e05dfa137..2097f638a 100644--- a/pkg/sources/stdin/stdin.go+++ b/pkg/sources/stdin/stdin.go@@ -60,7 +60,7 @@ func (s *Source) Chunks(ctx context.Context, chunksChan chan *sources.Chunk, _ .SourceMetadata: &source_metadatapb.MetaData{Data: &source_metadatapb.MetaData_Stdin{},},- Verify: s.verify,+ SourceVerify: s.verify,}ctx.Logger().Info("scanning stdin for secrets")diff --git a/pkg/sources/syslog/syslog.go b/pkg/sources/syslog/syslog.goindex 62f741f79..dc1a1af6c 100644--- a/pkg/sources/syslog/syslog.go+++ b/pkg/sources/syslog/syslog.go@@ -282,7 +282,7 @@ func (s *Source) monitorConnection(ctx context.Context, conn net.Conn, chunksChaJobID: s.JobID(),SourceMetadata: metadata,Data: input,- Verify: s.verify,+ SourceVerify: s.verify,}}}@@ -332,7 +332,7 @@ func (s *Source) acceptUDPConnections(ctx context.Context, netListener net.PackeSourceType: s.syslog.sourceType,SourceMetadata: metadata,Data: input,- Verify: s.verify,+ SourceVerify: s.verify,}}}diff --git a/pkg/sources/travisci/travisci.go b/pkg/sources/travisci/travisci.goindex a7ead20d0..6065bddd6 100644--- a/pkg/sources/travisci/travisci.go+++ b/pkg/sources/travisci/travisci.go@@ -203,7 +203,7 @@ func (s *Source) ChunkUnit(ctx context.Context, unit sources.SourceUnit, reporte},},},- Verify: s.verify,+ SourceVerify: s.verify,}if err := reporter.ChunkOk(ctx, chunk); err != nil {