Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Base64 decoding depth assessment (#4744)
Dylan Ayrey952df702b3e5
diff --git a/docs/iterative_decoding_performance.md b/docs/iterative_decoding_performance.mdnew file mode 100644index 000000000..5c51c8b60--- /dev/null+++ b/docs/iterative_decoding_performance.md@@ -0,0 +1,67 @@+# Iterative Decoding Performance++Performance characteristics of the `--max-decode-depth` feature, which enables+chained decoding (e.g., base64 inside UTF-16, double-encoded base64).++## How it works++At depth 0, all decoders run on the original chunk (identical to pre-existing+behavior). When a decoder produces new output, that output is fed back through+all decoders at the next depth level. The loop exits early when no decoder+produces new data, so unused depth levels are effectively free.++The PLAIN (UTF-8) decoder is skipped at depth > 0 since it's a passthrough+that never transforms data produced by other decoders (their output is already+valid UTF-8/ASCII).++## Filesystem scan benchmark++Scanned the trufflehog repository (~4,500 files) with `--no-verification`+and `--concurrency=1` for deterministic comparison.++| Depth | Wall time | Unique results | Delta vs depth=1 |+|-------|-----------|----------------|-------------------|+| 1 | 8.05s | 924 | — |+| 2 | 8.18s | 927 | +3, +1.6% |+| 3 | 8.09s | 928 | +4, +0.5% |+| 5 | 8.19s | 928 | +4, +1.7% |+| 10 | 8.35s | 932 | +8, +3.7% |++Results converge by depth 3. Depths 4–5 produce no additional decoded data in+this corpus, so they add only a single `len() == 0` check per chunk per extra+depth level.++The small unique-result variance at depth 10 is from pre-existing+nondeterminism in the concurrent detector workers' dedup ordering, not from the+decoding itself.++## Per-decoder microbenchmarks++Individual decoder cost is unchanged by this feature (decoders are not+modified). For reference, base64 decoder latency on random data:++| Input size | Latency/op | Allocs |+|------------|------------|-----------|+| 100 B | ~250 ns | 96 B / 2 |+| 1 KB | ~2.25 µs | 96 B / 2 |+| 10 KB | ~44 ns | 96 B / 2 |++The 10 KB case is fast because random bytes rarely form valid base64 substrings+(the 20-character minimum threshold is never met), so the decoder exits after a+single O(n) character scan.++## Memory overhead++Each depth level that produces new decoded data stores one copy of the output+(typically smaller than the input, since base64 decoding shrinks by ~25%).+A `seen` list (slice of byte slices) prevents reprocessing identical data.+At depth 5 on a typical chunk, this list has 0–3 entries. No hashing or maps+are used.++## Choosing a depth++| Depth | Use case |+|-------|----------|+| 1 | Legacy behavior, no chaining |+| 2 | Covers base64-in-base64, base64-in-UTF-16, base64-in-escaped-unicode |+| 5 | Default. Handles deeply nested configs with no measurable cost over depth 2 |diff --git a/main.go b/main.goindex 93a2a9825..bbe1b44e9 100644--- a/main.go+++ b/main.go@@ -65,6 +65,7 @@ var (filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()scanEntireChunk = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool()+ maxDecodeDepth = cli.Flag("max-decode-depth", "Maximum depth of iterative decoding. Each decoder's output is fed back through all decoders, up to this limit. 1 = single pass, 2+ = chained decoding (e.g., base64 inside utf16).").Default("5").Int()compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool()configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile()// rules = cli.Flag("rules", "Path to file with custom rules.").String()@@ -562,6 +563,7 @@ func run(state overseer.State, logSync func() error) {Results: parsedResults,PrintAvgDetectorTime: *printAvgDetectorTime,ShouldScanEntireChunk: *scanEntireChunk,+ MaxDecodeDepth: *maxDecodeDepth,VerificationCacheMetrics: &verificationCacheMetrics,}diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex f8bc54709..e45651dca 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -8,6 +8,7 @@ import ("errors""fmt""runtime"+ "slices""strconv""sync""sync/atomic"@@ -153,6 +154,12 @@ type Config struct {VerificationResultCache verificationcache.ResultCacheVerificationCacheMetrics verificationcache.MetricsReporter++ // MaxDecodeDepth is the maximum number of iterative decoding passes per chunk.+ // When a decoder transforms data, all decoders are re-run on the output up to this limit.+ // 1 = single pass (no chaining), 2+ = chained (e.g., base64 inside UTF-16).+ // Default: 5.+ MaxDecodeDepth int}// Engine represents the core scanning engine responsible for detecting secrets in input data.@@ -221,6 +228,8 @@ type Engine struct {notificationWorkerMultiplier int// verificationOverlapWorkerMultiplier is used to calculate the number of verification overlap workers.verificationOverlapWorkerMultiplier int++ maxDecodeDepth int}// NewEngine creates a new Engine instance with the provided configuration.@@ -245,6 +254,7 @@ func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {detectorWorkerMultiplier: cfg.DetectorWorkerMultiplier,notificationWorkerMultiplier: cfg.NotificationWorkerMultiplier,verificationOverlapWorkerMultiplier: cfg.VerificationOverlapWorkerMultiplier,+ maxDecodeDepth: cfg.MaxDecodeDepth,}if engine.sourceManager == nil {return nil, fmt.Errorf("source manager is required")@@ -354,6 +364,10 @@ func (e *Engine) setDefaults(ctx context.Context) {e.verificationOverlapWorkerMultiplier = 1}+ if e.maxDecodeDepth < 1 {+ e.maxDecodeDepth = 1+ }+// Default decoders handle common encoding formats.if len(e.decoders) == 0 {e.decoders = decoders.DefaultDecoders()@@ -775,6 +789,62 @@ type verificationOverlapChunk struct {verificationOverlapWgDoneFn func()}+// iterativeDecode applies all decoders to data, then re-applies them to any+// new output, up to maxDepth passes. Each pass skips the PLAIN (UTF-8) decoder+// because all other decoders already produce valid UTF-8/ASCII output, so+// re-running PLAIN would only duplicate work without changing the data.+//+// The returned chunks include results from every depth level -- intermediate+// decoded forms are scanned, not just the final one, because a secret may only+// be recognizable at a particular decoding stage.+func iterativeDecode(chunk *sources.Chunk, allDecoders []decoders.Decoder, maxDepth int) []*decoders.DecodableChunk {+ var results []*decoders.DecodableChunk++ currentInputs := [][]byte{chunk.Data}+ var seen [][]byte++ for depth := 0; depth < maxDepth; depth++ {+ var nextInputs [][]byte++ for _, data := range currentInputs {+ for _, decoder := range allDecoders {+ // The PLAIN (UTF-8) decoder always returns non-nil and only transforms+ // invalid UTF-8 via extractSubstrings. All other decoders already produce+ // valid UTF-8/ASCII output, so re-running PLAIN at depth > 0 would just+ // duplicate detector work without ever changing the data.+ if depth > 0 && decoder.Type() == detectorspb.DecoderType_PLAIN {+ continue+ }++ chunkCopy := *chunk+ chunkCopy.Data = data+ decoded := decoder.FromChunk(&chunkCopy)+ if decoded == nil {+ continue+ }++ results = append(results, decoded)++ if depth+1 < maxDepth &&+ decoder.Type() != detectorspb.DecoderType_PLAIN &&+ !bytes.Equal(decoded.Chunk.Data, data) &&+ !slices.ContainsFunc(seen, func(s []byte) bool { return bytes.Equal(s, decoded.Chunk.Data) }) {++ seen = append(seen, decoded.Chunk.Data)+ nextInputs = append(nextInputs, decoded.Chunk.Data)+ }+ }+ }++ if len(nextInputs) == 0 {+ break+ }+ currentInputs = nextInputs+ }++ return results+}+func (e *Engine) scannerWorker(ctx context.Context) {var wgDetect sync.WaitGroupvar wgVerificationOverlap sync.WaitGroup@@ -782,38 +852,29 @@ func (e *Engine) scannerWorker(ctx context.Context) {for chunk := range e.ChunksChan() {startTime := time.Now()sourceVerify := chunk.Verify- for _, decoder := range e.decoders {- decodeStart := time.Now()- // This copy is needed to preserve the original chunk.Data across multiple decoders.- chunkCopy := *chunk- decoded := decoder.FromChunk(&chunkCopy)- decodeTime := time.Since(decodeStart).Microseconds()- decodeLatency.WithLabelValues(decoder.Type().String(), chunk.SourceName).Observe(float64(decodeTime))-- if decoded == nil {- // This means that the decoder didn't understand this chunk and isn't applicable to it.- continue- }- matchingDetectors := e.AhoCorasickCore.FindDetectorMatches(decoded.Chunk.Data)+ decoded := iterativeDecode(chunk, e.decoders, e.maxDecodeDepth)++ for _, d := range decoded {+ matchingDetectors := e.AhoCorasickCore.FindDetectorMatches(d.Chunk.Data)if len(matchingDetectors) > 1 && !e.verificationOverlap {wgVerificationOverlap.Add(1)e.verificationOverlapChunksChan <- verificationOverlapChunk{- chunk: *decoded.Chunk,+ chunk: *d.Chunk,detectors: matchingDetectors,- decoder: decoded.DecoderType,+ decoder: d.DecoderType,verificationOverlapWgDoneFn: wgVerificationOverlap.Done,}continue}for _, detector := range matchingDetectors {- decoded.Chunk.Verify = e.shouldVerifyChunk(sourceVerify, detector, e.detectorVerificationOverrides)+ d.Chunk.Verify = e.shouldVerifyChunk(sourceVerify, detector, e.detectorVerificationOverrides)wgDetect.Add(1)e.detectableChunksChan <- detectableChunk{- chunk: *decoded.Chunk,+ chunk: *d.Chunk,detector: detector,- decoder: decoded.DecoderType,+ decoder: d.DecoderType,wgDoneFn: wgDetect.Done,}}diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.goindex ed46cba9e..ec4df8603 100644--- a/pkg/engine/engine_test.go+++ b/pkg/engine/engine_test.go@@ -1548,6 +1548,7 @@ func TestEngine_ScannerWorker_DetectableChunkHasCorrectVerifyFlag(t *testing.T)detectableChunksChan: make(chan detectableChunk, 1),sourceManager: sources.NewManager(),verify: true,+ maxDecodeDepth: 1,}// Arrange: Create a chunk to scan.@@ -1686,3 +1687,104 @@ func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(t}})}++func TestEngine_IterativeDecoding(t *testing.T) {+ t.Parallel()++ // base64(base64("my-secret-key-test-value"))+ const (+ doubleEncoded = "YlhrdGMyVmpjbVYwTFd0bGVTMTBaWE4wTFhaaGJIVmw="+ detectorKeyword = "my-secret"+ )++ // "token: bXktc2VjcmV0LWtleS10ZXN0LXZhbHVl end" as UTF-16LE+ utf16ContainingBase64 := []byte{+ 116, 0, 111, 0, 107, 0, 101, 0, 110, 0, 58, 0, 32, 0,+ 98, 0, 88, 0, 107, 0, 116, 0, 99, 0, 50, 0, 86, 0, 106, 0,+ 99, 0, 109, 0, 86, 0, 48, 0, 76, 0, 87, 0, 116, 0, 108, 0,+ 101, 0, 83, 0, 49, 0, 48, 0, 90, 0, 88, 0, 78, 0, 48, 0,+ 76, 0, 88, 0, 90, 0, 104, 0, 98, 0, 72, 0, 86, 0, 108, 0,+ 32, 0, 101, 0, 110, 0, 100, 0,+ }++ tests := []struct {+ name string+ input []byte+ depth int+ wantKeyword bool+ }{+ {+ name: "double base64, depth=1, miss",+ input: []byte("token: " + doubleEncoded),+ depth: 1,+ wantKeyword: false,+ },+ {+ name: "double base64, depth=2, found",+ input: []byte("token: " + doubleEncoded),+ depth: 2,+ wantKeyword: true,+ },+ {+ name: "utf16+base64, depth=1, miss",+ input: utf16ContainingBase64,+ depth: 1,+ wantKeyword: false,+ },+ {+ name: "utf16+base64, depth=2, found",+ input: utf16ContainingBase64,+ depth: 2,+ wantKeyword: true,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ t.Parallel()+ ctx := context.Background()++ detector := &passthroughDetector{+ keywords: []string{detectorKeyword},+ detectorType: detectorspb.DetectorType(9999),+ }+ e := &Engine{+ AhoCorasickCore: ahocorasick.NewAhoCorasickCore([]detectors.Detector{detector}),+ decoders: decoders.DefaultDecoders(),+ detectableChunksChan: make(chan detectableChunk, 64),+ sourceManager: sources.NewManager(),+ maxDecodeDepth: tt.depth,+ }++ e.sourceManager.ScanChunk(&sources.Chunk{Data: tt.input})+ go e.scannerWorker(ctx)++ var found bool+ timeout := time.After(2 * time.Second)+ Loop:+ for {+ select {+ case dc := <-e.detectableChunksChan:+ dc.wgDoneFn()+ found = true+ for {+ select {+ case dc2 := <-e.detectableChunksChan:+ dc2.wgDoneFn()+ case <-time.After(200 * time.Millisecond):+ break Loop+ }+ }+ case <-timeout:+ break Loop+ }+ }++ if tt.wantKeyword {+ assert.True(t, found, "expected detector match")+ } else {+ assert.False(t, found, "unexpected detector match")+ }+ })+ }+}diff --git a/pkg/engine/metrics.go b/pkg/engine/metrics.goindex 212061ca9..f2f43b3cc 100644--- a/pkg/engine/metrics.go+++ b/pkg/engine/metrics.go@@ -8,17 +8,6 @@ import ()var (- decodeLatency = promauto.NewHistogramVec(- prometheus.HistogramOpts{- Namespace: common.MetricsNamespace,- Subsystem: common.MetricsSubsystem,- Name: "decode_latency",- Help: "Time spent decoding a chunk in microseconds",- Buckets: prometheus.ExponentialBuckets(50, 2, 20),- },- []string{"decoder_type", "source_name"},- )-// Detector metrics.detectorExecutionCount = promauto.NewCounterVec(prometheus.CounterOpts{