Skip to content

Find, verify, and analyze leaked credentials

[INT-1084] Make the results deduplication cache size configurable (#5399)

Charlie Gunyon005b692bdfd0
diff --git a/docs/man/trufflehog.1 b/docs/man/trufflehog.1
index b417a7041..ed9e2a872 100644
--- a/docs/man/trufflehog.1
+++ b/docs/man/trufflehog.1
@@ -50,6 +50,9 @@ Don't verify the results.
\fB--results=RESULTS\fR
Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.
.TP
+\fB--deduplication-cache-size=5000\fR
+Max size of the results deduplication cache, setting to 0 disables
+.TP
\fB--no-color\fR
Disable colorized output
.TP
diff --git a/main.go b/main.go
index 001d7ae32..0b3d72002 100644
--- a/main.go
+++ b/main.go
@@ -60,8 +60,13 @@ var (
noVerification = cli.Flag("no-verification", "Don't verify the results.").Bool()
onlyVerified = cli.Flag("only-verified", "Only output verified results.").Hidden().Bool()
results = cli.Flag("results", "Specifies which type(s) of results to output: verified (confirmed valid by API), unknown (verification failed due to error), unverified (detected but not verified), filtered_unverified (unverified but would have been filtered out). Defaults to verified,unverified,unknown.").String()
- noColor = cli.Flag("no-color", "Disable colorized output").Bool()
- noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool()
+
+ // The default cache size is 5000 entries, which is a balance between memory usage and the need for effective deduplication.
+ // Since the cache entries are md5 hashes so each entry would be 16 bytes, so in total this would be aorund 80KB of memory usage.
+ dedupeCacheSize = cli.Flag("deduplication-cache-size", "Max size of the results deduplication cache, setting to 0 disables").Default("5000").Int()
+
+ noColor = cli.Flag("no-color", "Disable colorized output").Bool()
+ noColour = cli.Flag("no-colour", "Alias for --no-color").Hidden().Bool()
allowVerificationOverlap = cli.Flag("allow-verification-overlap", "Allow verification of similar credentials across detectors").Bool()
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
@@ -669,6 +674,7 @@ func run(state overseer.State, logSync func() error) {
ShouldScanEntireChunk: *scanEntireChunk,
MaxDecodeDepth: *maxDecodeDepth,
VerificationCacheMetrics: &verificationCacheMetrics,
+ DedupeCacheSize: *dedupeCacheSize,
}
if !*noVerificationCache {
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 6764b5a89..c549a9448 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -179,6 +179,9 @@ type Config struct {
// 1 = single pass (no chaining), 2+ = chained (e.g., base64 inside UTF-16).
// Default: 5.
MaxDecodeDepth int
+
+ // Max size of the deduplication LRU cache
+ DedupeCacheSize int
}
// Engine represents the core scanning engine responsible for detecting secrets in input data.
@@ -252,6 +255,8 @@ type Engine struct {
maxDecodeDepth int
+ dedupeCacheSize int
+
// runtimeCollector exposes live channel/worker/scan counters to Prometheus
// while the engine is running. Set in Start, cleared in Finish.
runtimeCollector *runtimeCollector
@@ -569,14 +574,15 @@ func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors
// deduplication efforts, allowing the engine to quickly check if a chunk has
// been processed before, thereby saving computational overhead.
func (e *Engine) initialize(ctx context.Context) error {
- // The cache size is set to 5000 entries, which is a balance between memory usage and the need for effective deduplication.
- // Since the cache entries are md5 hashes so each entry would be 16 bytes, so in total this would be aorund 80KB of memory usage.
- const cacheSize = 5000
+ if e.dedupeCacheEnabled() {
+ cache, err := lru.New[string, struct{}](e.dedupeCacheSize)
+ if err != nil {
+ return fmt.Errorf("failed to initialize LRU cache: %w", err)
+ }
- cache, err := lru.New[string, struct{}](cacheSize)
- if err != nil {
- return fmt.Errorf("failed to initialize LRU cache: %w", err)
+ e.dedupeCache = cache
}
+
const (
// detectableChunksChanMultiplier is set to accommodate a high number of concurrent worker goroutines.
// This multiplier ensures that the detectableChunksChan channel has sufficient buffer capacity
@@ -600,7 +606,6 @@ func (e *Engine) initialize(ctx context.Context) error {
chan verificationOverlapChunk, defaultChannelBuffer*verificationOverlapChunksChanMultiplier,
)
e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer*resultsChanMultiplier)
- e.dedupeCache = cache
ctx.Logger().V(4).Info("engine initialized")
// Configure the EntireChunkSpanCalculator if the engine is set to scan the entire chunk.
@@ -616,6 +621,10 @@ func (e *Engine) initialize(ctx context.Context) error {
return nil
}
+func (e *Engine) dedupeCacheEnabled() bool {
+ return e.dedupeCacheSize > 0
+}
+
const ignoreTag = "trufflehog:ignore"
// AhoCorasickCoreKeywords returns a set of keywords that the engine's
@@ -1410,7 +1419,7 @@ func (e *Engine) notifierWorker(ctx context.Context) {
// result from reverification and want to Dispatch it below.
// Notifier workers share this cache; the check and insert must be one atomic step.
- if result.SecretID == 0 {
+ if e.dedupeCacheEnabled() && result.SecretID == 0 {
h := md5.Sum([]byte(fmt.Sprintf("%s%s%s%s%+v", result.DetectorName, result.DetectorType.String(), result.Raw, result.RawV2, result.SourceMetadata)))
key := string(h[:])
if found, _ := e.dedupeCache.ContainsOrAdd(key, struct{}{}); found {
diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go
index c7c6419ac..0612a31ff 100644
--- a/pkg/engine/engine_test.go
+++ b/pkg/engine/engine_test.go
@@ -2295,6 +2295,7 @@ func TestNotifierWorker_ReverifiedResultsBypassDedupe(t *testing.T) {
notifyVerifiedResults: true,
notifyUnverifiedResults: true,
notifyUnknownResults: true,
+ dedupeCacheSize: 5000,
}
result := detectors.ResultWithMetadata{
@@ -2352,6 +2353,7 @@ func TestNotifierWorker_ConcurrentDuplicatesDispatchedOnce(t *testing.T) {
notifyVerifiedResults: true,
notifyUnverifiedResults: true,
notifyUnknownResults: true,
+ dedupeCacheSize: 5000,
}
// Fill and close the channel before starting workers so they all contend