Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Implement verification cache (#3801)
Cody Roseddc015e5ed99
diff --git a/main.go b/main.goindex 4c5cf2982..9f7a66597 100644--- a/main.go+++ b/main.go@@ -20,6 +20,9 @@ import ("github.com/go-logr/logr""github.com/jpillora/overseer""github.com/mattn/go-isatty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache""go.uber.org/automaxprocs/maxprocs""github.com/trufflesecurity/trufflehog/v3/pkg/analyzer"@@ -76,6 +79,8 @@ var (excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()jobReportFile = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)+ noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool()+// Add feature flagsforceSkipBinaries = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool()forceSkipArchives = cli.Flag("force-skip-archives", "Force skipping archives.").Bool()@@ -480,25 +485,32 @@ func run(state overseer.State) {logFatal(err, "failed to configure results flag")}+ verificationCacheMetrics := verificationcache.InMemoryMetrics{}+engConf := engine.Config{Concurrency: *concurrency,// The engine must always be configured with the list of// default detectors, which can be further filtered by the// user. The filters are applied by the engine and are only// subtractive.- Detectors: append(defaults.DefaultDetectors(), conf.Detectors...),- Verify: !*noVerification,- IncludeDetectors: *includeDetectors,- ExcludeDetectors: *excludeDetectors,- CustomVerifiersOnly: *customVerifiersOnly,- VerifierEndpoints: *verifiers,- Dispatcher: engine.NewPrinterDispatcher(printer),- FilterUnverified: *filterUnverified,- FilterEntropy: *filterEntropy,- VerificationOverlap: *allowVerificationOverlap,- Results: parsedResults,- PrintAvgDetectorTime: *printAvgDetectorTime,- ShouldScanEntireChunk: *scanEntireChunk,+ Detectors: append(defaults.DefaultDetectors(), conf.Detectors...),+ Verify: !*noVerification,+ IncludeDetectors: *includeDetectors,+ ExcludeDetectors: *excludeDetectors,+ CustomVerifiersOnly: *customVerifiersOnly,+ VerifierEndpoints: *verifiers,+ Dispatcher: engine.NewPrinterDispatcher(printer),+ FilterUnverified: *filterUnverified,+ FilterEntropy: *filterEntropy,+ VerificationOverlap: *allowVerificationOverlap,+ Results: parsedResults,+ PrintAvgDetectorTime: *printAvgDetectorTime,+ ShouldScanEntireChunk: *scanEntireChunk,+ VerificationCacheMetrics: &verificationCacheMetrics,+ }++ if !*noVerificationCache {+ engConf.VerificationResultCache = simple.NewCache[detectors.Result]()}if *compareDetectionStrategies {@@ -518,6 +530,20 @@ func run(state overseer.State) {logFatal(err, "error running scan")}+ verificationCacheMetrics := struct {+ Hits int32+ Misses int32+ HitsWasted int32+ AttemptsSaved int32+ VerificationTimeSpentMS int64+ }{+ Hits: verificationCacheMetrics.ResultCacheHits.Load(),+ Misses: verificationCacheMetrics.ResultCacheMisses.Load(),+ HitsWasted: verificationCacheMetrics.ResultCacheHitsWasted.Load(),+ AttemptsSaved: verificationCacheMetrics.CredentialVerificationsSaved.Load(),+ VerificationTimeSpentMS: verificationCacheMetrics.FromDataVerifyTimeSpentMS.Load(),+ }+// Print results.logger.Info("finished scanning","chunks", metrics.ChunksScanned,@@ -526,6 +552,7 @@ func run(state overseer.State) {"unverified_secrets", metrics.UnverifiedSecretsFound,"scan_duration", metrics.ScanDuration.String(),"trufflehog_version", version.BuildVersion,+ "verification_caching", verificationCacheMetrics,)if metrics.hasFoundResults && *fail {diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex de05193f2..9b3a4ffed 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -90,6 +90,9 @@ type Result struct {// DetectorName is the name of the Detector. Used for custom detectors.DetectorName stringVerified bool+ // VerificationFromCache indicates whether this result's verification result came from the verification cache rather+ // than an actual remote request.+ VerificationFromCache bool// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.Raw []byte// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.@@ -111,7 +114,15 @@ type Result struct {AnalysisInfo map[string]string}-// SetVerificationError is the only way to set a verification error. Any sensitive values should be passed-in as secrets to be redacted.+// CopyVerificationInfo clones verification info (status and error) from another Result struct. This is used when+// loading verification info from a verification cache. (A method is necessary because verification errors are not+// exported, to prevent the accidental storage of sensitive information in them.)+func (r *Result) CopyVerificationInfo(from *Result) {+ r.Verified = from.Verified+ r.verificationError = from.verificationError+}++// SetVerificationError is the only way to set a new verification error. Any sensitive values should be passed-in as secrets to be redacted.func (r *Result) SetVerificationError(err error, secrets ...string) {if err != nil {r.verificationError = redactSecrets(err, secrets...)diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 7d856c84a..0ee4080f4 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -13,6 +13,7 @@ import ("github.com/adrg/strutil""github.com/adrg/strutil/metrics"lru "github.com/hashicorp/golang-lru/v2"+ "github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache""google.golang.org/protobuf/proto""github.com/trufflesecurity/trufflehog/v3/pkg/common"@@ -145,6 +146,9 @@ type Config struct {// VerificationOverlapWorkerMultiplier is used to determine the number of verification overlap workers to spawn.VerificationOverlapWorkerMultiplier int++ VerificationResultCache verificationcache.ResultCache+ VerificationCacheMetrics verificationcache.MetricsReporter}// Engine represents the core scanning engine responsible for detecting secrets in input data.@@ -153,9 +157,10 @@ type Config struct {// customization through various options and configurations.type Engine struct {// CLI flags.- concurrency int- decoders []decoders.Decoder- detectors []detectors.Detector+ concurrency int+ decoders []decoders.Decoder+ detectors []detectors.Detector+ verificationCache *verificationcache.VerificationCache// Any detectors configured to override sources' verification flagsdetectorVerificationOverrides map[config.DetectorID]bool@@ -216,10 +221,13 @@ type Engine struct {// NewEngine creates a new Engine instance with the provided configuration.func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {+ verificationCache := verificationcache.New(cfg.VerificationResultCache, cfg.VerificationCacheMetrics)+engine := &Engine{concurrency: cfg.Concurrency,decoders: cfg.Decoders,detectors: cfg.Detectors,+ verificationCache: verificationCache,dispatcher: cfg.Dispatcher,verify: cfg.Verify,filterUnverified: cfg.FilterUnverified,@@ -1056,7 +1064,12 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {t := time.AfterFunc(detectionTimeout+1*time.Second, func() {ctx.Logger().Error(nil, "a detector ignored the context timeout")})- results, err := data.detector.Detector.FromData(ctx, data.chunk.Verify, matchBytes)+ results, err := e.verificationCache.FromData(+ ctx,+ data.detector.Detector,+ data.chunk.Verify,+ data.chunk.SecretID != 0,+ matchBytes)t.Stop()cancel()if err != nil {diff --git a/pkg/output/json.go b/pkg/output/json.goindex d98e78909..8a99d3206 100644--- a/pkg/output/json.go+++ b/pkg/output/json.go@@ -40,9 +40,10 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)// DetectorDescription is the description of the Detector.DetectorDescription string// DecoderName is the string name of the DecoderType.- DecoderName string- Verified bool- VerificationError string `json:",omitempty"`+ DecoderName string+ Verified bool+ VerificationError string `json:",omitempty"`+ VerificationFromCache bool// Raw contains the raw secret data.Raw string// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.@@ -54,21 +55,22 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)ExtraData map[string]stringStructuredData *detectorspb.StructuredData}{- SourceMetadata: r.SourceMetadata,- SourceID: r.SourceID,- SourceType: r.SourceType,- SourceName: r.SourceName,- DetectorType: r.DetectorType,- DetectorName: r.DetectorType.String(),- DetectorDescription: r.DetectorDescription,- DecoderName: r.DecoderType.String(),- Verified: r.Verified,- VerificationError: verificationErr,- Raw: string(r.Raw),- RawV2: string(r.RawV2),- Redacted: r.Redacted,- ExtraData: r.ExtraData,- StructuredData: r.StructuredData,+ SourceMetadata: r.SourceMetadata,+ SourceID: r.SourceID,+ SourceType: r.SourceType,+ SourceName: r.SourceName,+ DetectorType: r.DetectorType,+ DetectorName: r.DetectorType.String(),+ DetectorDescription: r.DetectorDescription,+ DecoderName: r.DecoderType.String(),+ Verified: r.Verified,+ VerificationError: verificationErr,+ VerificationFromCache: r.VerificationFromCache,+ Raw: string(r.Raw),+ RawV2: string(r.RawV2),+ Redacted: r.Redacted,+ ExtraData: r.ExtraData,+ StructuredData: r.StructuredData,}out, err := json.Marshal(v)if err != nil {diff --git a/pkg/output/plain.go b/pkg/output/plain.goindex c6a987161..f67d9268a 100644--- a/pkg/output/plain.go+++ b/pkg/output/plain.go@@ -22,6 +22,7 @@ var (boldGreenPrinter = color.New(color.Bold, color.FgHiGreen)whitePrinter = color.New(color.FgWhite)boldWhitePrinter = color.New(color.Bold, color.FgWhite)+ cyanPrinter = color.New(color.FgCyan))// PlainPrinter is a printer that prints results in plain text format.@@ -56,6 +57,9 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)yellowPrinter.Printf("Verification issue: %s\n", out.VerificationError)}}+ if r.VerificationFromCache {+ cyanPrinter.Print("(Verification info cached)\n")+ }printer.Printf("Detector Type: %s\n", out.DetectorType)printer.Printf("Decoder Type: %s\n", out.DecoderType)printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))diff --git a/pkg/verificationcache/in_memory_metrics.go b/pkg/verificationcache/in_memory_metrics.gonew file mode 100644index 000000000..e5a79eb93--- /dev/null+++ b/pkg/verificationcache/in_memory_metrics.go@@ -0,0 +1,37 @@+package verificationcache++import (+ "sync/atomic"+ "time"+)++// InMemoryMetrics is a MetricsReporter that stores reported metrics in memory for retrieval at the end of a scan.+type InMemoryMetrics struct {+ CredentialVerificationsSaved atomic.Int32+ FromDataVerifyTimeSpentMS atomic.Int64+ ResultCacheHits atomic.Int32+ ResultCacheHitsWasted atomic.Int32+ ResultCacheMisses atomic.Int32+}++var _ MetricsReporter = (*InMemoryMetrics)(nil)++func (m *InMemoryMetrics) AddCredentialVerificationsSaved(count int) {+ m.CredentialVerificationsSaved.Add(int32(count))+}++func (m *InMemoryMetrics) AddFromDataVerifyTimeSpent(wallTime time.Duration) {+ m.FromDataVerifyTimeSpentMS.Add(wallTime.Milliseconds())+}++func (m *InMemoryMetrics) AddResultCacheHits(count int) {+ m.ResultCacheHits.Add(int32(count))+}++func (m *InMemoryMetrics) AddResultCacheMisses(count int) {+ m.ResultCacheMisses.Add(int32(count))+}++func (m *InMemoryMetrics) AddResultCacheHitsWasted(count int) {+ m.ResultCacheHitsWasted.Add(int32(count))+}diff --git a/pkg/verificationcache/metrics_reporter.go b/pkg/verificationcache/metrics_reporter.gonew file mode 100644index 000000000..c5f16865c--- /dev/null+++ b/pkg/verificationcache/metrics_reporter.go@@ -0,0 +1,28 @@+package verificationcache++import "time"++// MetricsReporter is an interface used by a verification cache to report various metrics related to its operation.+// Implementations must be thread-safe.+type MetricsReporter interface {+ // AddCredentialVerificationsSaved records "saved" verification attempts, which is when credential verification+ // status is loaded from the cache instead of retrieved from a remote verification endpoint. This number might be+ // smaller than the cache hit count due to cache hit "wasting"; see AddResultCacheHitsWasted for more information.+ AddCredentialVerificationsSaved(count int)++ // AddFromDataVerifyTimeSpent records wall time spent in calls to detector.FromData with verify=true.+ AddFromDataVerifyTimeSpent(wallTime time.Duration)++ // AddResultCacheHits records result cache hits. Not all cache hits result in elided remote verification requests+ // due to cache hit "wasting"; see AddResultCacheHitsWasted for more information.+ AddResultCacheHits(count int)++ // AddResultCacheMisses records result cache misses.+ AddResultCacheMisses(count int)++ // AddResultCacheHitsWasted records "wasted" result cache hits. A "wasted" result cache hit is a result cache hit+ // that does not elide a remote verification request because there are other secret findings in the relevant chunk+ // that are not cached. When this happens, the detector's FromData method must be called anyway, so the cache hit+ // doesn't save any remote requests.+ AddResultCacheHitsWasted(count int)+}diff --git a/pkg/verificationcache/result_cache.go b/pkg/verificationcache/result_cache.gonew file mode 100644index 000000000..b915f65f3--- /dev/null+++ b/pkg/verificationcache/result_cache.go@@ -0,0 +1,9 @@+package verificationcache++import (+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+)++// ResultCache is a cache that holds individual detector results. It serves as a component of a VerificationCache.+type ResultCache cache.Cache[detectors.Result]diff --git a/pkg/verificationcache/verification_cache.go b/pkg/verificationcache/verification_cache.gonew file mode 100644index 000000000..8f2c9f609--- /dev/null+++ b/pkg/verificationcache/verification_cache.go@@ -0,0 +1,139 @@+package verificationcache++import (+ "sync"+ "time"++ "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/hasher"+)++// VerificationCache is a structure that can be used to cache verification results from detectors so that a given+// credential does not trigger multiple identical remote verification attempts.+type VerificationCache struct {+ metrics MetricsReporter+ resultCache ResultCache++ hashMu sync.Mutex+ hasher hasher.Hasher+}++// New creates a new verification cache with the provided result cache and metrics reporter. If resultCache is nil, the+// verification cache will be a no-op passthrough, although it will still record relevant metrics to the provided+// metrics reporter in this case.+func New(resultCache ResultCache, metrics MetricsReporter) *VerificationCache {+ if metrics == nil {+ metrics = &InMemoryMetrics{}+ }++ return &VerificationCache{+ metrics: metrics,+ resultCache: resultCache,+ hasher: hasher.NewBlake2B(),+ }+}++// FromData is a cache-aware facade in front of the provided detector's FromData method.+//+// If the verification cache's underlying result cache is nil, or verify is false, or forceCacheUpdate is true, this+// method invokes the provided detector's FromData method with the provided arguments and returns the result. If the+// result cache is non-nil and forceCacheUpdate is true, the result cache is updated with the results before they are+// returned.+//+// Otherwise, the detector's FromData method is called with verify=false. The result cache is then checked for each+// returned result. If there is a cache hit for each result, these cached values are all returned. Otherwise, the+// detector's FromData method is called again, but with verify=true, and the results are stored in the cache and then+// returned.+func (v *VerificationCache) FromData(+ ctx context.Context,+ detector detectors.Detector,+ verify bool,+ forceCacheUpdate bool,+ data []byte,+) ([]detectors.Result, error) {++ if v.resultCache == nil {+ if verify {+ start := time.Now()+ defer func() {+ v.metrics.AddFromDataVerifyTimeSpent(time.Since(start))+ }()+ }++ return detector.FromData(ctx, verify, data)+ }++ if !forceCacheUpdate {+ withoutRemoteVerification, err := detector.FromData(ctx, false, data)+ if err != nil {+ return nil, err+ }++ if !verify {+ return withoutRemoteVerification, nil+ }++ isEverythingCached := true+ var cacheHitsInCurrentChunk int+ for i, r := range withoutRemoteVerification {+ cacheKey, err := v.getResultCacheKey(r)+ if err != nil {+ ctx.Logger().Error(err, "error getting result cache key for verification caching",+ "operation", "read")+ isEverythingCached = false+ v.metrics.AddResultCacheHitsWasted(cacheHitsInCurrentChunk)+ break+ }+ if cacheHit, ok := v.resultCache.Get(string(cacheKey)); ok {+ withoutRemoteVerification[i].CopyVerificationInfo(&cacheHit)+ withoutRemoteVerification[i].VerificationFromCache = true+ v.metrics.AddResultCacheHits(1)+ cacheHitsInCurrentChunk+++ } else {+ v.metrics.AddResultCacheMisses(1)+ isEverythingCached = false+ v.metrics.AddResultCacheHitsWasted(cacheHitsInCurrentChunk)+ break+ }+ }++ if isEverythingCached {+ v.metrics.AddCredentialVerificationsSaved(len(withoutRemoteVerification))+ return withoutRemoteVerification, nil+ }+ }++ start := time.Now()+ withRemoteVerification, err := detector.FromData(ctx, verify, data)+ if verify {+ v.metrics.AddFromDataVerifyTimeSpent(time.Since(start))+ }+ if err != nil {+ return nil, err+ }++ for _, r := range withRemoteVerification {+ cacheKey, err := v.getResultCacheKey(r)+ if err != nil {+ ctx.Logger().Error(err, "error getting result cache key for verification caching",+ "operation", "write")+ continue+ }++ copyForCaching := r+ // Do not persist raw secret values in a long-lived cache+ copyForCaching.Raw = nil+ copyForCaching.RawV2 = nil+ v.resultCache.Set(string(cacheKey), copyForCaching)+ }++ return withRemoteVerification, nil+}++func (v *VerificationCache) getResultCacheKey(result detectors.Result) ([]byte, error) {+ v.hashMu.Lock()+ defer v.hashMu.Unlock()++ return v.hasher.Hash(append(result.Raw, result.RawV2...))+}diff --git a/pkg/verificationcache/verification_cache_test.go b/pkg/verificationcache/verification_cache_test.gonew file mode 100644index 000000000..69476844d--- /dev/null+++ b/pkg/verificationcache/verification_cache_test.go@@ -0,0 +1,240 @@+package verificationcache++import (+ "context"+ "errors"+ "testing"+ "time"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"+ logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++type testDetector struct {+ fromDataCallCount int+ results []detectors.Result+}++func (t *testDetector) FromData(_ context.Context, verify bool, _ []byte) ([]detectors.Result, error) {+ t.fromDataCallCount = t.fromDataCallCount + 1+ var results []detectors.Result+ for _, r := range t.results {+ copy := detectors.Result{Redacted: r.Redacted, Raw: r.Raw, RawV2: r.RawV2}+ if verify {+ copy.CopyVerificationInfo(&r)+ }+ results = append(results, copy)+ }++ // The metric timing resolution is 1 ms, so the detector needs to artificially slow down so that it can actually be+ // monitored.+ time.Sleep(2 * time.Millisecond)++ return results, nil+}++func (t *testDetector) Keywords() []string { return nil }+func (t *testDetector) Type() detectorspb.DetectorType { return -1 }+func (t *testDetector) Description() string { return "" }++var _ detectors.Detector = (*testDetector)(nil)++func getResultCacheKey(t *testing.T, cache *VerificationCache, result detectors.Result) string {+ key, err := cache.getResultCacheKey(result)+ require.NoError(t, err)+ return string(key)+}++func TestVerificationCacheFromData_Passthrough(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ }}++ require.NotPanics(t, func() {+ cache := New(nil, nil)+ results, err := cache.FromData(+ logContext.Background(),+ &detector,+ true,+ true,+ nil)++ require.NoError(t, err)+ assert.Equal(t, 1, detector.fromDataCallCount)+ assert.ElementsMatch(t, detector.results, results)+ })+}++func TestVerificationCacheFromData_VerifyFalseForceCacheUpdateFalse(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ }}+ metrics := InMemoryMetrics{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)++ results, err := cache.FromData(+ logContext.Background(),+ &detector,+ false,+ false,+ nil)++ require.NoError(t, err)+ assert.Equal(t, 1, detector.fromDataCallCount)+ assert.ElementsMatch(t, []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: false},+ }, results)+ assert.Empty(t, cache.resultCache.Values())+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+ assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheHits.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load())+}++func TestFromDataCached_VerifyFalseForceCacheUpdateTrue(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }}+ detector.results[1].SetVerificationError(errors.New("test verification error"))+ metrics := InMemoryMetrics{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)++ results, err := cache.FromData(+ logContext.Background(),+ &detector,+ false,+ true,+ nil)++ require.NoError(t, err)+ assert.Equal(t, 1, detector.fromDataCallCount)+ assert.ElementsMatch(t, []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: false},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }, results)+ assert.ElementsMatch(t, []detectors.Result{+ {Redacted: "hello", Verified: false},+ {Redacted: "world", Verified: false},+ }, cache.resultCache.Values())+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+ assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheHits.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load())+}++func TestFromDataCached_VerifyTrueForceCacheUpdateFalseAllCacheHits(t *testing.T) {+ remoteResults := []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }+ remoteResults[1].SetVerificationError(errors.New("test verification error"))+ detector := testDetector{results: remoteResults}+ cacheData := []detectors.Result{+ {Redacted: "hello", Verified: false},+ {Redacted: "world", Verified: true},+ }+ cacheData[0].SetVerificationError(errors.New("test verification error"))+ metrics := InMemoryMetrics{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)+ cache.resultCache.Set(getResultCacheKey(t, cache, remoteResults[0]), cacheData[0])+ cache.resultCache.Set(getResultCacheKey(t, cache, remoteResults[1]), cacheData[1])++ results, err := cache.FromData(+ logContext.Background(),+ &detector,+ true,+ false,+ nil)++ require.NoError(t, err)+ assert.Equal(t, 1, detector.fromDataCallCount)+ wantResults := []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: false, VerificationFromCache: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: true, VerificationFromCache: true},+ }+ wantResults[0].SetVerificationError(errors.New("test verification error"))+ assert.ElementsMatch(t, wantResults, results)+ assert.ElementsMatch(t, cacheData, cache.resultCache.Values())+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+ assert.Equal(t, int32(2), metrics.CredentialVerificationsSaved.Load())+ assert.Equal(t, int32(2), metrics.ResultCacheHits.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load())+}++func TestFromDataCached_VerifyTrueForceCacheUpdateFalseCacheMiss(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }}+ detector.results[1].SetVerificationError(errors.New("test verification error"))+ cachedResult := detectors.Result{Redacted: "hello", Verified: false}+ cachedResult.SetVerificationError(errors.New("test verification error"))+ resultCache := simple.NewCache[detectors.Result]()+ metrics := InMemoryMetrics{}+ cache := New(resultCache, &metrics)+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), cachedResult)++ results, err := cache.FromData(+ logContext.Background(),+ &detector,+ true,+ false,+ nil)++ require.NoError(t, err)+ assert.Equal(t, 2, detector.fromDataCallCount)+ assert.ElementsMatch(t, detector.results, results)+ wantCacheData := []detectors.Result{+ {Redacted: "hello", Verified: true},+ {Redacted: "world", Verified: false},+ }+ wantCacheData[1].SetVerificationError(errors.New("test verification error"))+ assert.ElementsMatch(t, wantCacheData, cache.resultCache.Values())+ assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+ assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load())+ assert.Equal(t, int32(1), metrics.ResultCacheHits.Load())+ assert.Equal(t, int32(1), metrics.ResultCacheMisses.Load())+ assert.Equal(t, int32(1), metrics.ResultCacheHitsWasted.Load())+}++func TestFromDataCached_VerifyTrueForceCacheUpdateTrue(t *testing.T) {+ detector := testDetector{results: []detectors.Result{+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},+ }}+ detector.results[1].SetVerificationError(errors.New("test verification error"))+ metrics := InMemoryMetrics{}+ cache := New(simple.NewCache[detectors.Result](), &metrics)+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), detectors.Result{Redacted: "hello", Verified: false})+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[1]), detectors.Result{Redacted: "world", Verified: true})++ results, err := cache.FromData(+ logContext.Background(),+ &detector,+ true,+ true,+ nil)++ require.NoError(t, err)+ assert.Equal(t, 1, detector.fromDataCallCount)+ assert.ElementsMatch(t, detector.results, results)+ wantCacheData := []detectors.Result{+ {Redacted: "hello", Verified: true},+ {Redacted: "world", Verified: false},+ }+ wantCacheData[1].SetVerificationError(errors.New("test verification error"))+ assert.ElementsMatch(t, wantCacheData, cache.resultCache.Values())+ assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())+ assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheHits.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load())+ assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load())+}