Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add per detector verification timing to verification cache (#5341)
Kevin Pham7ee4d4970dde
diff --git a/pkg/verificationcache/metrics_reporter.go b/pkg/verificationcache/metrics_reporter.goindex 9ed2f1172..3a9974266 100644--- a/pkg/verificationcache/metrics_reporter.go+++ b/pkg/verificationcache/metrics_reporter.go@@ -1,6 +1,10 @@package verificationcache-import "time"+import (+ "time"++ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"+)// MetricsReporter is an interface used by a verification cache to report various metrics related to its operation.// Implementations must be thread-safe.@@ -28,3 +32,19 @@ type MetricsReporter interface {// doesn't save any remote requests.AddResultCacheHitsWasted(count int)}++// DetectorMetricsReporter is an optional interface that a MetricsReporter can additionally implement to receive+// verification timing attributed to the detector that incurred it. Implementations must be thread-safe.+//+// What one sample covers depends on the path taken. When the result cache is consulted, a detectors.ResultVerifier+// records one sample per remote VerifyResult call on a cache miss. Every other case records one sample per whole+// FromData(verify=true) pass, including its regex pass: detectors that do not implement detectors.ResultVerifier,+// targeted rescans (forceCacheUpdate), and a VerificationCache with no result cache. The last two apply to+// ResultVerifier detectors too, so one such detector can produce whole-pass samples that cover many remote calls+// alongside its per-call samples. A detector that starts implementing detectors.ResultVerifier shifts its+// distribution, which resets its baseline rather than signaling a provider change.+type DetectorMetricsReporter interface {+ // AddDetectorVerifyTimeSpent records wall time spent verifying credentials remotely for one detector. It is not+ // called for results served from the cache.+ AddDetectorVerifyTimeSpent(detectorType detector_typepb.DetectorType, wallTime time.Duration)+}diff --git a/pkg/verificationcache/verification_cache.go b/pkg/verificationcache/verification_cache.goindex 494c7e64b..83d9528bd 100644--- a/pkg/verificationcache/verification_cache.go+++ b/pkg/verificationcache/verification_cache.go@@ -9,13 +9,15 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/hasher"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb")// VerificationCache is a structure that can be used to cache verification results from detectors so that a given// credential does not trigger multiple identical remote verification attempts.type VerificationCache struct {- metrics MetricsReporter- resultCache ResultCache+ metrics MetricsReporter+ resultCache ResultCache+ detectorMetrics DetectorMetricsReporter // optionalhashMu sync.Mutexhasher hasher.Hasher@@ -28,11 +30,31 @@ func New(resultCache ResultCache, metrics MetricsReporter) *VerificationCache {if metrics == nil {metrics = &InMemoryMetrics{}}+ detectorMetrics, _ := metrics.(DetectorMetricsReporter)return &VerificationCache{- metrics: metrics,- resultCache: resultCache,- hasher: hasher.NewBlake2B(),+ metrics: metrics,+ resultCache: resultCache,+ detectorMetrics: detectorMetrics,+ hasher: hasher.NewBlake2B(),+ }+}++// recordVerifyTime reports one remote verification pass to both metric views: the cross-detector aggregate and, when+// the reporter opted in, the per-detector histogram. Used by the paths where a single detector.FromData call with+// verify=true is the whole verification pass.+func (v *VerificationCache) recordVerifyTime(detectorType detector_typepb.DetectorType, wallTime time.Duration) {+ v.metrics.AddFromDataVerifyTimeSpent(wallTime)+ v.recordDetectorVerifyTime(detectorType, wallTime)+}++// recordDetectorVerifyTime reports one verification sample to the per-detector view. It is a no-op for reporters that+// do not implement DetectorMetricsReporter, which keeps existing MetricsReporter implementations unaffected.+// verifyCacheMisses calls it directly because it samples each VerifyResult call per detector but reports the aggregate+// once per chunk.+func (v *VerificationCache) recordDetectorVerifyTime(detectorType detector_typepb.DetectorType, wallTime time.Duration) {+ if v.detectorMetrics != nil {+ v.detectorMetrics.AddDetectorVerifyTimeSpent(detectorType, wallTime)}}@@ -62,7 +84,7 @@ func (v *VerificationCache) FromData(if verify {start := time.Now()defer func() {- v.metrics.AddFromDataVerifyTimeSpent(time.Since(start))+ v.recordVerifyTime(detector.Type(), time.Since(start))}()}@@ -82,7 +104,7 @@ func (v *VerificationCache) FromData(// avoiding re-running verification for every result in the chunk.// only if a detector implements detectors.ResultVerifierif resultVerifier, ok := detector.(detectors.ResultVerifier); ok {- return v.verifyCacheMisses(ctx, resultVerifier, withoutRemoteVerification)+ return v.verifyCacheMisses(ctx, resultVerifier, detector.Type(), withoutRemoteVerification)}isEverythingCached := truevar cacheHitsInCurrentChunk int@@ -117,7 +139,7 @@ func (v *VerificationCache) FromData(start := time.Now()withRemoteVerification, err := detector.FromData(ctx, verify, data)if verify {- v.metrics.AddFromDataVerifyTimeSpent(time.Since(start))+ v.recordVerifyTime(detector.Type(), time.Since(start))}if err != nil {return nil, err@@ -145,10 +167,12 @@ func (v *VerificationCache) FromData(func (v *VerificationCache) verifyCacheMisses(ctx context.Context,detector detectors.ResultVerifier,+ detectorType detector_typepb.DetectorType,results []detectors.Result,) ([]detectors.Result, error) {- // Only remote verification counts toward verify time; a fully cached chunk records- // nothing, matching the all-or-nothing path's early return on full cache coverage.+ // Only remote verification is timed: cache hits never reach verifyResult, so a fully cached chunk records nothing.+ // The aggregate is summed and reported once per chunk because MetricsReporter implementations such as+ // InMemoryMetrics truncate each report to whole milliseconds, which would drop time if reported per call.var timeSpentVerifying time.Durationdefer func() {if timeSpentVerifying > 0 {@@ -158,7 +182,9 @@ func (v *VerificationCache) verifyCacheMisses(verifyResult := func(i int) {verifyStart := time.Now()detector.VerifyResult(ctx, &results[i])- timeSpentVerifying += time.Since(verifyStart)+ elapsed := time.Since(verifyStart)+ v.recordDetectorVerifyTime(detectorType, elapsed)+ timeSpentVerifying += elapsed}for i := range results {diff --git a/pkg/verificationcache/verification_cache_test.go b/pkg/verificationcache/verification_cache_test.goindex 81ea780c6..3946a2e04 100644--- a/pkg/verificationcache/verification_cache_test.go+++ b/pkg/verificationcache/verification_cache_test.go@@ -3,6 +3,7 @@ package verificationcacheimport ("context""errors"+ "sync""testing""time"@@ -17,6 +18,9 @@ import (type testDetector struct {fromDataCallCount intresults []detectors.Result+ // fromDataErr, when set, is returned by FromData instead of results, so tests can exercise what+ // happens when remote verification fails rather than merely returning an unverified credential.+ fromDataErr error}func (t *testDetector) FromData(_ context.Context, verify bool, _ []byte) ([]detectors.Result, error) {@@ -43,6 +47,12 @@ func (t *testDetector) FromData(_ context.Context, verify bool, _ []byte) ([]det// monitored.time.Sleep(2 * time.Millisecond)+ // Failing after the sleep keeps the elapsed time observable, which is what the error-path timing+ // test needs to assert.+ if t.fromDataErr != nil {+ return nil, t.fromDataErr+ }+return results, nil}@@ -87,6 +97,94 @@ var (_ detectors.ResultVerifier = (*testResultVerifier)(nil))+// typedTestDetector reports a detector type chosen by the test rather than the fixed -1 of+// testDetector, so tests can prove that verification timing is attributed to the detector that+// incurred it and not merely recorded.+type typedTestDetector struct {+ testDetector+ detectorType detector_typepb.DetectorType+}++func (t *typedTestDetector) Type() detector_typepb.DetectorType { return t.detectorType }++var _ detectors.Detector = (*typedTestDetector)(nil)++// detectorVerifySample is one observation handed to DetectorMetricsReporter.+type detectorVerifySample struct {+ detectorType detector_typepb.DetectorType+ wallTime time.Duration+}++// detectorMetricsRecorder is an InMemoryMetrics that additionally opts into per-detector+// verification timing. Keeping the aggregate behavior by embedding is deliberate: these tests have+// to show that the new per-detector view is added without disturbing the existing counter, so both+// have to be observable through one reporter, exactly as thog's reporter will see them.+type detectorMetricsRecorder struct {+ InMemoryMetrics++ mu sync.Mutex+ // samples accumulates per-detector observations in call order.+ samples []detectorVerifySample+ // aggregateCalls counts calls to AddFromDataVerifyTimeSpent. The sum alone cannot distinguish+ // one report of an accumulated total from several per-call reports, and that distinction is the+ // point of the per-remote-call granularity tests.+ aggregateCalls int+}++var (+ _ MetricsReporter = (*detectorMetricsRecorder)(nil)+ _ DetectorMetricsReporter = (*detectorMetricsRecorder)(nil)+)++func (m *detectorMetricsRecorder) AddFromDataVerifyTimeSpent(wallTime time.Duration) {+ m.mu.Lock()+ m.aggregateCalls+++ m.mu.Unlock()++ m.InMemoryMetrics.AddFromDataVerifyTimeSpent(wallTime)+}++func (m *detectorMetricsRecorder) AddDetectorVerifyTimeSpent(+ detectorType detector_typepb.DetectorType,+ wallTime time.Duration,+) {+ m.mu.Lock()+ defer m.mu.Unlock()++ m.samples = append(m.samples, detectorVerifySample{detectorType: detectorType, wallTime: wallTime})+}++func (m *detectorMetricsRecorder) recordedSamples() []detectorVerifySample {+ m.mu.Lock()+ defer m.mu.Unlock()++ return append([]detectorVerifySample(nil), m.samples...)+}++func (m *detectorMetricsRecorder) aggregateReportCount() int {+ m.mu.Lock()+ defer m.mu.Unlock()++ return m.aggregateCalls+}++// assertSampleTypes asserts that the recorded samples carry exactly the given detector types, in+// order, and that each one reports observable wall time. A sample of zero duration would satisfy a+// count-only assertion while carrying no latency signal at all.+func assertSampleTypes(t *testing.T, recorder *detectorMetricsRecorder, want ...detector_typepb.DetectorType) {+ t.Helper()++ samples := recorder.recordedSamples()+ // Left nil rather than preallocated so that the no-samples case compares equal to a caller that+ // passed no expected types at all.+ var gotTypes []detector_typepb.DetectorType+ for _, sample := range samples {+ gotTypes = append(gotTypes, sample.detectorType)+ assert.Greater(t, sample.wallTime, time.Duration(0))+ }+ assert.Equal(t, want, gotTypes)+}+func getResultCacheKey(t *testing.T, cache *VerificationCache, result detectors.Result) string {key, err := cache.getResultCacheKey(result)require.NoError(t, err)@@ -573,3 +671,211 @@ func TestVerificationCache_FromData_DoesNotCacheSecretMaterial(t *testing.T) {assert.Nil(t, cached[0].SecretParts)assert.Empty(t, cached[0].GetPrimarySecretValue())}++// The tests below cover per-detector verification timing, which a reporter receives only by also+// implementing DetectorMetricsReporter. They exist to catch two classes of regression: a sample+// attributed to the wrong detector, and a sample recorded for verification that never left the+// process because the result cache answered it.++func TestVerificationCache_FromData_PerDetectorTiming_Passthrough(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ }}+ metrics := detectorMetricsRecorder{}+ cache := New(nil, &metrics)++ _, err := cache.FromData(logContext.Background(), &detector, true, true, nil)++ require.NoError(t, err)+ assertSampleTypes(t, &metrics, detector.Type())+ assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+}++func TestVerificationCache_FromData_PerDetectorTiming_PassthroughVerifyFalse(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ }}+ metrics := detectorMetricsRecorder{}+ cache := New(nil, &metrics)++ _, err := cache.FromData(logContext.Background(), &detector, false, true, nil)++ require.NoError(t, err)+ assertSampleTypes(t, &metrics)+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+}++func TestVerificationCache_FromData_PerDetectorTiming_CacheMiss(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }}+ metrics := detectorMetricsRecorder{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]),+ detectors.Result{Redacted: "hello", Verified: true})++ _, err := cache.FromData(logContext.Background(), &detector, true, false, nil)++ require.NoError(t, err)+ // The uncached finding forces one FromData(verify=true) pass over the whole chunk, which is one+ // verification pass and therefore one sample.+ assertSampleTypes(t, &metrics, detector.Type())+}++// A chunk answered entirely from the cache issues no remote request, so recording a duration for it+// would put fabricated latency into the histogram.+func TestVerificationCache_FromData_PerDetectorTiming_AllCacheHits(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }}+ metrics := detectorMetricsRecorder{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]),+ detectors.Result{Redacted: "hello", Verified: true})+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[1]),+ detectors.Result{Redacted: "world", Verified: false})++ _, err := cache.FromData(logContext.Background(), &detector, true, false, nil)++ require.NoError(t, err)+ assertSampleTypes(t, &metrics)+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+}++func TestVerificationCache_FromData_PerDetectorTiming_ForceCacheUpdate(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ }}+ metrics := detectorMetricsRecorder{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]),+ detectors.Result{Redacted: "hello", Verified: false})++ _, err := cache.FromData(logContext.Background(), &detector, true, true, nil)++ require.NoError(t, err)+ // Targeted rescans bypass the cache to get a fresh answer, so they do make a remote call.+ assertSampleTypes(t, &metrics, detector.Type())+}++func TestVerificationCache_FromData_PerDetectorTiming_ResultVerifierPartialCacheHit(t *testing.T) {+ detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }}}+ metrics := detectorMetricsRecorder{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]),+ detectors.Result{Redacted: "hello", Verified: true})++ _, err := cache.FromData(logContext.Background(), &detector, true, false, nil)++ require.NoError(t, err)+ // Only the miss is verified remotely, so the cache hit contributes no sample.+ require.Equal(t, 1, detector.verifyResultCallCount)+ assertSampleTypes(t, &metrics, detector.Type())+}++func TestVerificationCache_FromData_PerDetectorTiming_ResultVerifierAllCacheHits(t *testing.T) {+ detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }}}+ metrics := detectorMetricsRecorder{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]),+ detectors.Result{Redacted: "hello", Verified: true})+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[1]),+ detectors.Result{Redacted: "world", Verified: false})++ _, err := cache.FromData(logContext.Background(), &detector, true, false, nil)++ require.NoError(t, err)+ require.Equal(t, 0, detector.verifyResultCallCount)+ assertSampleTypes(t, &metrics)+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+}++// One sample per remote call is what makes the histogram readable as provider latency: a chunk with+// several misses must not collapse into a single fat observation. The aggregate counter keeps its+// existing shape of one accumulated report per chunk, because it truncates to whole milliseconds and+// would lose time if it were reported per call.+func TestVerificationCache_FromData_PerDetectorTiming_ResultVerifierSamplesPerRemoteCall(t *testing.T) {+ detector := testResultVerifier{testDetector: testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }}}+ metrics := detectorMetricsRecorder{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)++ _, err := cache.FromData(logContext.Background(), &detector, true, false, nil)++ require.NoError(t, err)+ require.Equal(t, 2, detector.verifyResultCallCount)+ assertSampleTypes(t, &metrics, detector.Type(), detector.Type())+ assert.Equal(t, 1, metrics.aggregateReportCount())+}++func TestVerificationCache_FromData_PerDetectorTiming_AttributesToOwnDetectorType(t *testing.T) {+ const (+ firstType detector_typepb.DetectorType = -10+ secondType detector_typepb.DetectorType = -11+ )+ first := typedTestDetector{+ testDetector: testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), Verified: true},+ }},+ detectorType: firstType,+ }+ second := typedTestDetector{+ testDetector: testDetector{results: []detectors.Result{+ {Redacted: "world", Raw: []byte("world"), Verified: true},+ }},+ detectorType: secondType,+ }+ metrics := detectorMetricsRecorder{}+ cache := New(nil, &metrics)++ _, err := cache.FromData(logContext.Background(), &first, true, true, nil)+ require.NoError(t, err)+ _, err = cache.FromData(logContext.Background(), &second, true, true, nil)+ require.NoError(t, err)++ assertSampleTypes(t, &metrics, firstType, secondType)+}++// A reporter that predates this interface, or simply does not want the extra series, must keep+// working: New leaves detectorMetrics nil and the timing sites have to tolerate that.+func TestVerificationCache_FromData_PerDetectorTiming_ReporterWithoutOptIn(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ }}+ metrics := InMemoryMetrics{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)+ require.Nil(t, cache.detectorMetrics)++ _, err := cache.FromData(logContext.Background(), &detector, true, false, nil)++ require.NoError(t, err)+ assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+}++// A provider that has started timing out or erroring is precisely the shift this metric exists to+// surface, so a failed verification still has to contribute its latency.+func TestVerificationCache_FromData_PerDetectorTiming_RecordsOnVerificationError(t *testing.T) {+ detector := testDetector{+ results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ },+ fromDataErr: errors.New("test verification failure"),+ }+ metrics := detectorMetricsRecorder{}+ cache := New(nil, &metrics)++ _, err := cache.FromData(logContext.Background(), &detector, true, true, nil)++ require.Error(t, err)+ assertSampleTypes(t, &metrics, detector.Type())+}