Skip to content

Find, verify, and analyze leaked credentials

Base64 decoding depth assessment (#4744)

Dylan Ayrey952df702b3e5
diff --git a/docs/iterative_decoding_performance.md b/docs/iterative_decoding_performance.md
new file mode 100644
index 000000000..5c51c8b60
--- /dev/null
+++ b/docs/iterative_decoding_performance.md
@@ -0,0 +1,67 @@
+# Iterative Decoding Performance
+
+Performance characteristics of the `--max-decode-depth` feature, which enables
+chained decoding (e.g., base64 inside UTF-16, double-encoded base64).
+
+## How it works
+
+At depth 0, all decoders run on the original chunk (identical to pre-existing
+behavior). When a decoder produces new output, that output is fed back through
+all decoders at the next depth level. The loop exits early when no decoder
+produces new data, so unused depth levels are effectively free.
+
+The PLAIN (UTF-8) decoder is skipped at depth > 0 since it's a passthrough
+that never transforms data produced by other decoders (their output is already
+valid UTF-8/ASCII).
+
+## Filesystem scan benchmark
+
+Scanned the trufflehog repository (~4,500 files) with `--no-verification`
+and `--concurrency=1` for deterministic comparison.
+
+| Depth | Wall time | Unique results | Delta vs depth=1 |
+|-------|-----------|----------------|-------------------|
+| 1 | 8.05s | 924 | — |
+| 2 | 8.18s | 927 | +3, +1.6% |
+| 3 | 8.09s | 928 | +4, +0.5% |
+| 5 | 8.19s | 928 | +4, +1.7% |
+| 10 | 8.35s | 932 | +8, +3.7% |
+
+Results converge by depth 3. Depths 4–5 produce no additional decoded data in
+this corpus, so they add only a single `len() == 0` check per chunk per extra
+depth level.
+
+The small unique-result variance at depth 10 is from pre-existing
+nondeterminism in the concurrent detector workers' dedup ordering, not from the
+decoding itself.
+
+## Per-decoder microbenchmarks
+
+Individual decoder cost is unchanged by this feature (decoders are not
+modified). For reference, base64 decoder latency on random data:
+
+| Input size | Latency/op | Allocs |
+|------------|------------|-----------|
+| 100 B | ~250 ns | 96 B / 2 |
+| 1 KB | ~2.25 µs | 96 B / 2 |
+| 10 KB | ~44 ns | 96 B / 2 |
+
+The 10 KB case is fast because random bytes rarely form valid base64 substrings
+(the 20-character minimum threshold is never met), so the decoder exits after a
+single O(n) character scan.
+
+## Memory overhead
+
+Each depth level that produces new decoded data stores one copy of the output
+(typically smaller than the input, since base64 decoding shrinks by ~25%).
+A `seen` list (slice of byte slices) prevents reprocessing identical data.
+At depth 5 on a typical chunk, this list has 0–3 entries. No hashing or maps
+are used.
+
+## Choosing a depth
+
+| Depth | Use case |
+|-------|----------|
+| 1 | Legacy behavior, no chaining |
+| 2 | Covers base64-in-base64, base64-in-UTF-16, base64-in-escaped-unicode |
+| 5 | Default. Handles deeply nested configs with no measurable cost over depth 2 |
diff --git a/main.go b/main.go
index 93a2a9825..bbe1b44e9 100644
--- a/main.go
+++ b/main.go
@@ -65,6 +65,7 @@ var (
filterUnverified = cli.Flag("filter-unverified", "Only output first unverified result per chunk per detector if there are more than one results.").Bool()
filterEntropy = cli.Flag("filter-entropy", "Filter unverified results with Shannon entropy. Start with 3.0.").Float64()
scanEntireChunk = cli.Flag("scan-entire-chunk", "Scan the entire chunk for secrets.").Hidden().Default("false").Bool()
+ maxDecodeDepth = cli.Flag("max-decode-depth", "Maximum depth of iterative decoding. Each decoder's output is fed back through all decoders, up to this limit. 1 = single pass, 2+ = chained decoding (e.g., base64 inside utf16).").Default("5").Int()
compareDetectionStrategies = cli.Flag("compare-detection-strategies", "Compare different detection strategies for matching spans").Hidden().Default("false").Bool()
configFilename = cli.Flag("config", "Path to configuration file.").ExistingFile()
// rules = cli.Flag("rules", "Path to file with custom rules.").String()
@@ -562,6 +563,7 @@ func run(state overseer.State, logSync func() error) {
Results: parsedResults,
PrintAvgDetectorTime: *printAvgDetectorTime,
ShouldScanEntireChunk: *scanEntireChunk,
+ MaxDecodeDepth: *maxDecodeDepth,
VerificationCacheMetrics: &verificationCacheMetrics,
}
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index f8bc54709..e45651dca 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -8,6 +8,7 @@ import (
"errors"
"fmt"
"runtime"
+ "slices"
"strconv"
"sync"
"sync/atomic"
@@ -153,6 +154,12 @@ type Config struct {
VerificationResultCache verificationcache.ResultCache
VerificationCacheMetrics verificationcache.MetricsReporter
+
+ // MaxDecodeDepth is the maximum number of iterative decoding passes per chunk.
+ // When a decoder transforms data, all decoders are re-run on the output up to this limit.
+ // 1 = single pass (no chaining), 2+ = chained (e.g., base64 inside UTF-16).
+ // Default: 5.
+ MaxDecodeDepth int
}
// Engine represents the core scanning engine responsible for detecting secrets in input data.
@@ -221,6 +228,8 @@ type Engine struct {
notificationWorkerMultiplier int
// verificationOverlapWorkerMultiplier is used to calculate the number of verification overlap workers.
verificationOverlapWorkerMultiplier int
+
+ maxDecodeDepth int
}
// NewEngine creates a new Engine instance with the provided configuration.
@@ -245,6 +254,7 @@ func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {
detectorWorkerMultiplier: cfg.DetectorWorkerMultiplier,
notificationWorkerMultiplier: cfg.NotificationWorkerMultiplier,
verificationOverlapWorkerMultiplier: cfg.VerificationOverlapWorkerMultiplier,
+ maxDecodeDepth: cfg.MaxDecodeDepth,
}
if engine.sourceManager == nil {
return nil, fmt.Errorf("source manager is required")
@@ -354,6 +364,10 @@ func (e *Engine) setDefaults(ctx context.Context) {
e.verificationOverlapWorkerMultiplier = 1
}
+ if e.maxDecodeDepth < 1 {
+ e.maxDecodeDepth = 1
+ }
+
// Default decoders handle common encoding formats.
if len(e.decoders) == 0 {
e.decoders = decoders.DefaultDecoders()
@@ -775,6 +789,62 @@ type verificationOverlapChunk struct {
verificationOverlapWgDoneFn func()
}
+// iterativeDecode applies all decoders to data, then re-applies them to any
+// new output, up to maxDepth passes. Each pass skips the PLAIN (UTF-8) decoder
+// because all other decoders already produce valid UTF-8/ASCII output, so
+// re-running PLAIN would only duplicate work without changing the data.
+//
+// The returned chunks include results from every depth level -- intermediate
+// decoded forms are scanned, not just the final one, because a secret may only
+// be recognizable at a particular decoding stage.
+func iterativeDecode(chunk *sources.Chunk, allDecoders []decoders.Decoder, maxDepth int) []*decoders.DecodableChunk {
+ var results []*decoders.DecodableChunk
+
+ currentInputs := [][]byte{chunk.Data}
+ var seen [][]byte
+
+ for depth := 0; depth < maxDepth; depth++ {
+ var nextInputs [][]byte
+
+ for _, data := range currentInputs {
+ for _, decoder := range allDecoders {
+ // The PLAIN (UTF-8) decoder always returns non-nil and only transforms
+ // invalid UTF-8 via extractSubstrings. All other decoders already produce
+ // valid UTF-8/ASCII output, so re-running PLAIN at depth > 0 would just
+ // duplicate detector work without ever changing the data.
+ if depth > 0 && decoder.Type() == detectorspb.DecoderType_PLAIN {
+ continue
+ }
+
+ chunkCopy := *chunk
+ chunkCopy.Data = data
+ decoded := decoder.FromChunk(&chunkCopy)
+ if decoded == nil {
+ continue
+ }
+
+ results = append(results, decoded)
+
+ if depth+1 < maxDepth &&
+ decoder.Type() != detectorspb.DecoderType_PLAIN &&
+ !bytes.Equal(decoded.Chunk.Data, data) &&
+ !slices.ContainsFunc(seen, func(s []byte) bool { return bytes.Equal(s, decoded.Chunk.Data) }) {
+
+ seen = append(seen, decoded.Chunk.Data)
+ nextInputs = append(nextInputs, decoded.Chunk.Data)
+ }
+ }
+ }
+
+ if len(nextInputs) == 0 {
+ break
+ }
+ currentInputs = nextInputs
+ }
+
+ return results
+}
+
func (e *Engine) scannerWorker(ctx context.Context) {
var wgDetect sync.WaitGroup
var wgVerificationOverlap sync.WaitGroup
@@ -782,38 +852,29 @@ func (e *Engine) scannerWorker(ctx context.Context) {
for chunk := range e.ChunksChan() {
startTime := time.Now()
sourceVerify := chunk.Verify
- for _, decoder := range e.decoders {
- decodeStart := time.Now()
- // This copy is needed to preserve the original chunk.Data across multiple decoders.
- chunkCopy := *chunk
- decoded := decoder.FromChunk(&chunkCopy)
- decodeTime := time.Since(decodeStart).Microseconds()
- decodeLatency.WithLabelValues(decoder.Type().String(), chunk.SourceName).Observe(float64(decodeTime))
-
- if decoded == nil {
- // This means that the decoder didn't understand this chunk and isn't applicable to it.
- continue
- }
- matchingDetectors := e.AhoCorasickCore.FindDetectorMatches(decoded.Chunk.Data)
+ decoded := iterativeDecode(chunk, e.decoders, e.maxDecodeDepth)
+
+ for _, d := range decoded {
+ matchingDetectors := e.AhoCorasickCore.FindDetectorMatches(d.Chunk.Data)
if len(matchingDetectors) > 1 && !e.verificationOverlap {
wgVerificationOverlap.Add(1)
e.verificationOverlapChunksChan <- verificationOverlapChunk{
- chunk: *decoded.Chunk,
+ chunk: *d.Chunk,
detectors: matchingDetectors,
- decoder: decoded.DecoderType,
+ decoder: d.DecoderType,
verificationOverlapWgDoneFn: wgVerificationOverlap.Done,
}
continue
}
for _, detector := range matchingDetectors {
- decoded.Chunk.Verify = e.shouldVerifyChunk(sourceVerify, detector, e.detectorVerificationOverrides)
+ d.Chunk.Verify = e.shouldVerifyChunk(sourceVerify, detector, e.detectorVerificationOverrides)
wgDetect.Add(1)
e.detectableChunksChan <- detectableChunk{
- chunk: *decoded.Chunk,
+ chunk: *d.Chunk,
detector: detector,
- decoder: decoded.DecoderType,
+ decoder: d.DecoderType,
wgDoneFn: wgDetect.Done,
}
}
diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go
index ed46cba9e..ec4df8603 100644
--- a/pkg/engine/engine_test.go
+++ b/pkg/engine/engine_test.go
@@ -1548,6 +1548,7 @@ func TestEngine_ScannerWorker_DetectableChunkHasCorrectVerifyFlag(t *testing.T)
detectableChunksChan: make(chan detectableChunk, 1),
sourceManager: sources.NewManager(),
verify: true,
+ maxDecodeDepth: 1,
}
// Arrange: Create a chunk to scan.
@@ -1686,3 +1687,104 @@ func TestEngine_VerificationOverlapWorker_DetectableChunkHasCorrectVerifyFlag(t
}
})
}
+
+func TestEngine_IterativeDecoding(t *testing.T) {
+ t.Parallel()
+
+ // base64(base64("my-secret-key-test-value"))
+ const (
+ doubleEncoded = "YlhrdGMyVmpjbVYwTFd0bGVTMTBaWE4wTFhaaGJIVmw="
+ detectorKeyword = "my-secret"
+ )
+
+ // "token: bXktc2VjcmV0LWtleS10ZXN0LXZhbHVl end" as UTF-16LE
+ utf16ContainingBase64 := []byte{
+ 116, 0, 111, 0, 107, 0, 101, 0, 110, 0, 58, 0, 32, 0,
+ 98, 0, 88, 0, 107, 0, 116, 0, 99, 0, 50, 0, 86, 0, 106, 0,
+ 99, 0, 109, 0, 86, 0, 48, 0, 76, 0, 87, 0, 116, 0, 108, 0,
+ 101, 0, 83, 0, 49, 0, 48, 0, 90, 0, 88, 0, 78, 0, 48, 0,
+ 76, 0, 88, 0, 90, 0, 104, 0, 98, 0, 72, 0, 86, 0, 108, 0,
+ 32, 0, 101, 0, 110, 0, 100, 0,
+ }
+
+ tests := []struct {
+ name string
+ input []byte
+ depth int
+ wantKeyword bool
+ }{
+ {
+ name: "double base64, depth=1, miss",
+ input: []byte("token: " + doubleEncoded),
+ depth: 1,
+ wantKeyword: false,
+ },
+ {
+ name: "double base64, depth=2, found",
+ input: []byte("token: " + doubleEncoded),
+ depth: 2,
+ wantKeyword: true,
+ },
+ {
+ name: "utf16+base64, depth=1, miss",
+ input: utf16ContainingBase64,
+ depth: 1,
+ wantKeyword: false,
+ },
+ {
+ name: "utf16+base64, depth=2, found",
+ input: utf16ContainingBase64,
+ depth: 2,
+ wantKeyword: true,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+
+ detector := &passthroughDetector{
+ keywords: []string{detectorKeyword},
+ detectorType: detectorspb.DetectorType(9999),
+ }
+ e := &Engine{
+ AhoCorasickCore: ahocorasick.NewAhoCorasickCore([]detectors.Detector{detector}),
+ decoders: decoders.DefaultDecoders(),
+ detectableChunksChan: make(chan detectableChunk, 64),
+ sourceManager: sources.NewManager(),
+ maxDecodeDepth: tt.depth,
+ }
+
+ e.sourceManager.ScanChunk(&sources.Chunk{Data: tt.input})
+ go e.scannerWorker(ctx)
+
+ var found bool
+ timeout := time.After(2 * time.Second)
+ Loop:
+ for {
+ select {
+ case dc := <-e.detectableChunksChan:
+ dc.wgDoneFn()
+ found = true
+ for {
+ select {
+ case dc2 := <-e.detectableChunksChan:
+ dc2.wgDoneFn()
+ case <-time.After(200 * time.Millisecond):
+ break Loop
+ }
+ }
+ case <-timeout:
+ break Loop
+ }
+ }
+
+ if tt.wantKeyword {
+ assert.True(t, found, "expected detector match")
+ } else {
+ assert.False(t, found, "unexpected detector match")
+ }
+ })
+ }
+}
diff --git a/pkg/engine/metrics.go b/pkg/engine/metrics.go
index 212061ca9..f2f43b3cc 100644
--- a/pkg/engine/metrics.go
+++ b/pkg/engine/metrics.go
@@ -8,17 +8,6 @@ import (
)
var (
- decodeLatency = promauto.NewHistogramVec(
- prometheus.HistogramOpts{
- Namespace: common.MetricsNamespace,
- Subsystem: common.MetricsSubsystem,
- Name: "decode_latency",
- Help: "Time spent decoding a chunk in microseconds",
- Buckets: prometheus.ExponentialBuckets(50, 2, 20),
- },
- []string{"decoder_type", "source_name"},
- )
-
// Detector metrics.
detectorExecutionCount = promauto.NewCounterVec(
prometheus.CounterOpts{