Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[INT-1084] Make the results deduplication cache size configurable (#5399)
Charlie Gunyon005b692bdfd0
diff --git a/docs/man/trufflehog.1 b/docs/man/trufflehog.1index b417a7041..ed9e2a872 100644--- a/docs/man/trufflehog.1+++ b/docs/man/trufflehog.1@@ -50,6 +50,9 @@ Don't verify the results.\fB--results=RESULTS\fRSpecifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown..TP+\fB--deduplication-cache-size=5000\fR+Max size of the results deduplication cache, setting to 0 disables+.TP\fB--no-color\fRDisable colorized output.TPdiff --git a/main.go b/main.goindex 001d7ae32..0b3d72002 100644--- a/main.go+++ b/main.go@@ -60,8 +60,13 @@ var (noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()onlyVerified = cli.Flag("only-verified", "Only output verified results.").Hidden().Bool()results = cli.Flag("results", "Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.").String()- noColor = cli.Flag("no-color", "Disable colorized output").Bool()- noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool()++ // The default cache size is 5000 entries, which is a balance between memory usage and the need for effective deduplication.+ // Since the cache entries are md5 hashes so each entry would be 16 bytes, so in total this would be aorund 80KB of memory usage.+ dedupeCacheSize = cli.Flag("deduplication-cache-size", "Max size of the results deduplication cache, setting to 0 disables").Default("5000").Int()++ noColor = cli.Flag("no-color", "Disable colorized output").Bool()+ noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool()allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()@@ -669,6 +674,7 @@ func run(state overseer.State, logSync func() error) {ShouldScanEntireChunk: *scanEntireChunk,MaxDecodeDepth: *maxDecodeDepth,VerificationCacheMetrics: &verificationCacheMetrics,+ DedupeCacheSize: *dedupeCacheSize,}if !*noVerificationCache {diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 6764b5a89..c549a9448 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -179,6 +179,9 @@ type Config struct {// 1 = single pass (no chaining), 2+ = chained (e.g., base64 inside UTF-16).// Default: 5.MaxDecodeDepth int++ // Max size of the deduplication LRU cache+ DedupeCacheSize int}// Engine represents the core scanning engine responsible for detecting secrets in input data.@@ -252,6 +255,8 @@ type Engine struct {maxDecodeDepth int+ dedupeCacheSize int+// runtimeCollector exposes live channel/worker/scan counters to Prometheus// while the engine is running. Set in Start, cleared in Finish.runtimeCollector *runtimeCollector@@ -569,14 +574,15 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors// deduplication efforts, allowing the engine to quickly check if a chunk has// been processed before, thereby saving computational overhead.func (e *Engine) initialize(ctx context.Context) error {- // The cache size is set to 5000 entries, which is a balance between memory usage and the need for effective deduplication.- // Since the cache entries are md5 hashes so each entry would be 16 bytes, so in total this would be aorund 80KB of memory usage.- const cacheSize = 5000+ if e.dedupeCacheEnabled() {+ cache, err := lru.New[string, struct{}](e.dedupeCacheSize)+ if err != nil {+ return fmt.Errorf("failed to initialize LRU cache: %w", err)+ }- cache, err := lru.New[string, struct{}](cacheSize)- if err != nil {- return fmt.Errorf("failed to initialize LRU cache: %w", err)+ e.dedupeCache = cache}+const (// detectableChunksChanMultiplier is set to accommodate a high number of concurrent worker goroutines.// This multiplier ensures that the detectableChunksChan channel has sufficient buffer capacity@@ -600,7 +606,6 @@ func (e *Engine) initialize(ctx context.Context) error {chan verificationOverlapChunk, defaultChannelBuffer*verificationOverlapChunksChanMultiplier,)e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer*resultsChanMultiplier)- e.dedupeCache = cachectx.Logger().V(4).Info("engine initialized")// Configure the EntireChunkSpanCalculator if the engine is set to scan the entire chunk.@@ -616,6 +621,10 @@ func (e *Engine) initialize(ctx context.Context) error {return nil}+func (e *Engine) dedupeCacheEnabled() bool {+ return e.dedupeCacheSize > 0+}+const ignoreTag = "trufflehog:ignore"// AhoCorasickCoreKeywords returns a set of keywords that the engine's@@ -1410,7 +1419,7 @@ func (e *Engine) notifierWorker(ctx context.Context) {// result from reverification and want to Dispatch it below.// Notifier workers share this cache; the check and insert must be one atomic step.- if result.SecretID == 0 {+ if e.dedupeCacheEnabled() && result.SecretID == 0 {h := md5.Sum([]byte(fmt.Sprintf("%s%s%s%s%+v", result.DetectorName, result.DetectorType.String(), result.Raw, result.RawV2, result.SourceMetadata)))key := string(h[:])if found, _ := e.dedupeCache.ContainsOrAdd(key, struct{}{}); found {diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.goindex c7c6419ac..0612a31ff 100644--- a/pkg/engine/engine_test.go+++ b/pkg/engine/engine_test.go@@ -2295,6 +2295,7 @@ func TestNotifierWorker_ReverifiedResultsBypassDedupe(t *testing.T) {notifyVerifiedResults: true,notifyUnverifiedResults: true,notifyUnknownResults: true,+ dedupeCacheSize: 5000,}result := detectors.ResultWithMetadata{@@ -2352,6 +2353,7 @@ func TestNotifierWorker_ConcurrentDuplicatesDispatchedOnce(t *testing.T) {notifyVerifiedResults: true,notifyUnverifiedResults: true,notifyUnknownResults: true,+ dedupeCacheSize: 5000,}// Fill and close the channel before starting workers so they all contend