Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Dedupe results (#1479)
Zachary Riceb48ac24c464d
diff --git a/pkg/decoders/decoders.go b/pkg/decoders/decoders.goindex 8a98dc0c4..52b4709e0 100644--- a/pkg/decoders/decoders.go+++ b/pkg/decoders/decoders.go@@ -6,6 +6,7 @@ import (func DefaultDecoders() []Decoder {return []Decoder{+ // UTF8 must be first for duplicate detection&UTF8{},&Base64{},&UTF16{},diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 454f1825c..d01ccde8c 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -2,6 +2,7 @@ package engineimport ("bytes"+ "fmt""reflect""runtime""strings"@@ -233,6 +234,22 @@ func (e *Engine) BytesScanned() uint64 {return e.bytesScanned}+func (e *Engine) dedupeAndSend(chunkResults []detectors.ResultWithMetadata) {+ dedupeMap := make(map[string]struct{})+ for _, result := range chunkResults {+ // dedupe by comparing the detector type, raw result, and source metadata+ // NOTE: in order for the PLAIN decoder to maintain precedence, make sure UTF8 is the first decoder in the+ // default decoders list+ key := fmt.Sprintf("%s%s%s%+v", result.DetectorType.String(), result.Raw, result.RawV2, result.SourceMetadata)+ if _, ok := dedupeMap[key]; ok {+ continue+ }+ dedupeMap[key] = struct{}{}+ e.results <- result+ }++}+func (e *Engine) DetectorAvgTime() map[string][]time.Duration {logger := context.Background().Logger()avgTime := map[string][]time.Duration{}@@ -257,6 +274,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {func (e *Engine) detectorWorker(ctx context.Context) {for originalChunk := range e.chunks {for chunk := range sources.Chunker(originalChunk) {+ var chunkResults []detectors.ResultWithMetadatamatchedKeywords := make(map[string]struct{})atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))for _, decoder := range e.decoders {@@ -273,21 +291,12 @@ func (e *Engine) detectorWorker(ctx context.Context) {decoderType = detectorspb.DecoderType_UNKNOWN}- original := chunk.Datadecoded := decoder.FromChunk(chunk)if decoded == nil {continue}- if decoded == nil ||- // check if the decoded data is similar "enough" to the original data. If it is, then we can skip scanning the decoded data as- // it's likely already picked up by the PLAIN decoder. See related issue: https://github.com/trufflesecurity/trufflehog/issues/1450- (decoded != nil &&- decoderType == detectorspb.DecoderType_BASE64 && common.BytesEqual(original, decoded.Data, 40)) {- continue- }-// build a map of all keywords that were matched in the chunkfor _, m := range e.prefilter.FindAll(string(decoded.Data)) {matchedKeywords[strings.ToLower(string(decoded.Data[m.Start():m.End()]))] = struct{}{}@@ -343,7 +352,7 @@ func (e *Engine) detectorWorker(ctx context.Context) {continue}result.DecoderType = decoderType- e.results <- detectors.CopyMetadata(resultChunk, result)+ chunkResults = append(chunkResults, detectors.CopyMetadata(resultChunk, result))}if len(results) > 0 {@@ -363,6 +372,7 @@ func (e *Engine) detectorWorker(ctx context.Context) {}}}+ e.dedupeAndSend(chunkResults)}atomic.AddUint64(&e.chunksScanned, 1)}diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.goindex 362cf9932..34924cdc4 100644--- a/pkg/engine/engine_test.go+++ b/pkg/engine/engine_test.go@@ -3,6 +3,7 @@ package engineimport ("testing"+ "github.com/trufflesecurity/trufflehog/v3/pkg/decoders""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/sources")@@ -62,3 +63,12 @@ func TestFragmentLineOffset(t *testing.T) {})}}++// Test to make sure that DefaultDecoders always returns the UTF8 decoder first.+// Technically a decoder test but we want this to run and fail in CI+func TestDefaultDecoders(t *testing.T) {+ ds := decoders.DefaultDecoders()+ if _, ok := ds[0].(*decoders.UTF8); !ok {+ t.Errorf("DefaultDecoders() = %v, expected UTF8 decoder to be first", ds)+ }+}