Skip to content

Find, verify, and analyze leaked credentials

Implement verification cache (#3801)

Cody Roseddc015e5ed99
diff --git a/main.go b/main.go
index 4c5cf2982..9f7a66597 100644
--- a/main.go
+++ b/main.go
@@ -20,6 +20,9 @@ import (
"github.com/go-logr/logr"
"github.com/jpillora/overseer"
"github.com/mattn/go-isatty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
"go.uber.org/automaxprocs/maxprocs"
"github.com/trufflesecurity/trufflehog/v3/pkg/analyzer"
@@ -76,6 +79,8 @@ var (
excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
jobReportFile = cli.Flag("output-report", "Write a scan report to the provided path.").Hidden().OpenFile(os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+ noVerificationCache = cli.Flag("no-verification-cache", "Disable verification caching").Bool()
+
// Add feature flags
forceSkipBinaries = cli.Flag("force-skip-binaries", "Force skipping binaries.").Bool()
forceSkipArchives = cli.Flag("force-skip-archives", "Force skipping archives.").Bool()
@@ -480,25 +485,32 @@ func run(state overseer.State) {
logFatal(err, "failed to configure results flag")
}
+ verificationCacheMetrics := verificationcache.InMemoryMetrics{}
+
engConf := engine.Config{
Concurrency: *concurrency,
// The engine must always be configured with the list of
// default detectors, which can be further filtered by the
// user. The filters are applied by the engine and are only
// subtractive.
- Detectors: append(defaults.DefaultDetectors(), conf.Detectors...),
- Verify: !*noVerification,
- IncludeDetectors: *includeDetectors,
- ExcludeDetectors: *excludeDetectors,
- CustomVerifiersOnly: *customVerifiersOnly,
- VerifierEndpoints: *verifiers,
- Dispatcher: engine.NewPrinterDispatcher(printer),
- FilterUnverified: *filterUnverified,
- FilterEntropy: *filterEntropy,
- VerificationOverlap: *allowVerificationOverlap,
- Results: parsedResults,
- PrintAvgDetectorTime: *printAvgDetectorTime,
- ShouldScanEntireChunk: *scanEntireChunk,
+ Detectors: append(defaults.DefaultDetectors(), conf.Detectors...),
+ Verify: !*noVerification,
+ IncludeDetectors: *includeDetectors,
+ ExcludeDetectors: *excludeDetectors,
+ CustomVerifiersOnly: *customVerifiersOnly,
+ VerifierEndpoints: *verifiers,
+ Dispatcher: engine.NewPrinterDispatcher(printer),
+ FilterUnverified: *filterUnverified,
+ FilterEntropy: *filterEntropy,
+ VerificationOverlap: *allowVerificationOverlap,
+ Results: parsedResults,
+ PrintAvgDetectorTime: *printAvgDetectorTime,
+ ShouldScanEntireChunk: *scanEntireChunk,
+ VerificationCacheMetrics: &verificationCacheMetrics,
+ }
+
+ if !*noVerificationCache {
+ engConf.VerificationResultCache = simple.NewCache[detectors.Result]()
}
if *compareDetectionStrategies {
@@ -518,6 +530,20 @@ func run(state overseer.State) {
logFatal(err, "error running scan")
}
+ verificationCacheMetrics := struct {
+ Hits int32
+ Misses int32
+ HitsWasted int32
+ AttemptsSaved int32
+ VerificationTimeSpentMS int64
+ }{
+ Hits: verificationCacheMetrics.ResultCacheHits.Load(),
+ Misses: verificationCacheMetrics.ResultCacheMisses.Load(),
+ HitsWasted: verificationCacheMetrics.ResultCacheHitsWasted.Load(),
+ AttemptsSaved: verificationCacheMetrics.CredentialVerificationsSaved.Load(),
+ VerificationTimeSpentMS: verificationCacheMetrics.FromDataVerifyTimeSpentMS.Load(),
+ }
+
// Print results.
logger.Info("finished scanning",
"chunks", metrics.ChunksScanned,
@@ -526,6 +552,7 @@ func run(state overseer.State) {
"unverified_secrets", metrics.UnverifiedSecretsFound,
"scan_duration", metrics.ScanDuration.String(),
"trufflehog_version", version.BuildVersion,
+ "verification_caching", verificationCacheMetrics,
)
if metrics.hasFoundResults && *fail {
diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go
index de05193f2..9b3a4ffed 100644
--- a/pkg/detectors/detectors.go
+++ b/pkg/detectors/detectors.go
@@ -90,6 +90,9 @@ type Result struct {
// DetectorName is the name of the Detector. Used for custom detectors.
DetectorName string
Verified bool
+ // VerificationFromCache indicates whether this result's verification result came from the verification cache rather
+ // than an actual remote request.
+ VerificationFromCache bool
// Raw contains the raw secret identifier data. Prefer IDs over secrets since it is used for deduping after hashing.
Raw []byte
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
@@ -111,7 +114,15 @@ type Result struct {
AnalysisInfo map[string]string
}
-// SetVerificationError is the only way to set a verification error. Any sensitive values should be passed-in as secrets to be redacted.
+// CopyVerificationInfo clones verification info (status and error) from another Result struct. This is used when
+// loading verification info from a verification cache. (A method is necessary because verification errors are not
+// exported, to prevent the accidental storage of sensitive information in them.)
+func (r *Result) CopyVerificationInfo(from *Result) {
+ r.Verified = from.Verified
+ r.verificationError = from.verificationError
+}
+
+// SetVerificationError is the only way to set a new verification error. Any sensitive values should be passed-in as secrets to be redacted.
func (r *Result) SetVerificationError(err error, secrets ...string) {
if err != nil {
r.verificationError = redactSecrets(err, secrets...)
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 7d856c84a..0ee4080f4 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -13,6 +13,7 @@ import (
"github.com/adrg/strutil"
"github.com/adrg/strutil/metrics"
lru "github.com/hashicorp/golang-lru/v2"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/verificationcache"
"google.golang.org/protobuf/proto"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
@@ -145,6 +146,9 @@ type Config struct {
// VerificationOverlapWorkerMultiplier is used to determine the number of verification overlap workers to spawn.
VerificationOverlapWorkerMultiplier int
+
+ VerificationResultCache verificationcache.ResultCache
+ VerificationCacheMetrics verificationcache.MetricsReporter
}
// Engine represents the core scanning engine responsible for detecting secrets in input data.
@@ -153,9 +157,10 @@ type Config struct {
// customization through various options and configurations.
type Engine struct {
// CLI flags.
- concurrency int
- decoders []decoders.Decoder
- detectors []detectors.Detector
+ concurrency int
+ decoders []decoders.Decoder
+ detectors []detectors.Detector
+ verificationCache *verificationcache.VerificationCache
// Any detectors configured to override sources' verification flags
detectorVerificationOverrides map[config.DetectorID]bool
@@ -216,10 +221,13 @@ type Engine struct {
// NewEngine creates a new Engine instance with the provided configuration.
func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {
+ verificationCache := verificationcache.New(cfg.VerificationResultCache, cfg.VerificationCacheMetrics)
+
engine := &Engine{
concurrency: cfg.Concurrency,
decoders: cfg.Decoders,
detectors: cfg.Detectors,
+ verificationCache: verificationCache,
dispatcher: cfg.Dispatcher,
verify: cfg.Verify,
filterUnverified: cfg.FilterUnverified,
@@ -1056,7 +1064,12 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
t := time.AfterFunc(detectionTimeout+1*time.Second, func() {
ctx.Logger().Error(nil, "a detector ignored the context timeout")
})
- results, err := data.detector.Detector.FromData(ctx, data.chunk.Verify, matchBytes)
+ results, err := e.verificationCache.FromData(
+ ctx,
+ data.detector.Detector,
+ data.chunk.Verify,
+ data.chunk.SecretID != 0,
+ matchBytes)
t.Stop()
cancel()
if err != nil {
diff --git a/pkg/output/json.go b/pkg/output/json.go
index d98e78909..8a99d3206 100644
--- a/pkg/output/json.go
+++ b/pkg/output/json.go
@@ -40,9 +40,10 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
// DetectorDescription is the description of the Detector.
DetectorDescription string
// DecoderName is the string name of the DecoderType.
- DecoderName string
- Verified bool
- VerificationError string `json:",omitempty"`
+ DecoderName string
+ Verified bool
+ VerificationError string `json:",omitempty"`
+ VerificationFromCache bool
// Raw contains the raw secret data.
Raw string
// RawV2 contains the raw secret identifier that is a combination of both the ID and the secret.
@@ -54,21 +55,22 @@ func (p *JSONPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
ExtraData map[string]string
StructuredData *detectorspb.StructuredData
}{
- SourceMetadata: r.SourceMetadata,
- SourceID: r.SourceID,
- SourceType: r.SourceType,
- SourceName: r.SourceName,
- DetectorType: r.DetectorType,
- DetectorName: r.DetectorType.String(),
- DetectorDescription: r.DetectorDescription,
- DecoderName: r.DecoderType.String(),
- Verified: r.Verified,
- VerificationError: verificationErr,
- Raw: string(r.Raw),
- RawV2: string(r.RawV2),
- Redacted: r.Redacted,
- ExtraData: r.ExtraData,
- StructuredData: r.StructuredData,
+ SourceMetadata: r.SourceMetadata,
+ SourceID: r.SourceID,
+ SourceType: r.SourceType,
+ SourceName: r.SourceName,
+ DetectorType: r.DetectorType,
+ DetectorName: r.DetectorType.String(),
+ DetectorDescription: r.DetectorDescription,
+ DecoderName: r.DecoderType.String(),
+ Verified: r.Verified,
+ VerificationError: verificationErr,
+ VerificationFromCache: r.VerificationFromCache,
+ Raw: string(r.Raw),
+ RawV2: string(r.RawV2),
+ Redacted: r.Redacted,
+ ExtraData: r.ExtraData,
+ StructuredData: r.StructuredData,
}
out, err := json.Marshal(v)
if err != nil {
diff --git a/pkg/output/plain.go b/pkg/output/plain.go
index c6a987161..f67d9268a 100644
--- a/pkg/output/plain.go
+++ b/pkg/output/plain.go
@@ -22,6 +22,7 @@ var (
boldGreenPrinter = color.New(color.Bold, color.FgHiGreen)
whitePrinter = color.New(color.FgWhite)
boldWhitePrinter = color.New(color.Bold, color.FgWhite)
+ cyanPrinter = color.New(color.FgCyan)
)
// PlainPrinter is a printer that prints results in plain text format.
@@ -56,6 +57,9 @@ func (p *PlainPrinter) Print(_ context.Context, r *detectors.ResultWithMetadata)
yellowPrinter.Printf("Verification issue: %s\n", out.VerificationError)
}
}
+ if r.VerificationFromCache {
+ cyanPrinter.Print("(Verification info cached)\n")
+ }
printer.Printf("Detector Type: %s\n", out.DetectorType)
printer.Printf("Decoder Type: %s\n", out.DecoderType)
printer.Printf("Raw result: %s\n", whitePrinter.Sprint(out.Raw))
diff --git a/pkg/verificationcache/in_memory_metrics.go b/pkg/verificationcache/in_memory_metrics.go
new file mode 100644
index 000000000..e5a79eb93
--- /dev/null
+++ b/pkg/verificationcache/in_memory_metrics.go
@@ -0,0 +1,37 @@
+package verificationcache
+
+import (
+ "sync/atomic"
+ "time"
+)
+
+// InMemoryMetrics is a MetricsReporter that stores reported metrics in memory for retrieval at the end of a scan.
+type InMemoryMetrics struct {
+ CredentialVerificationsSaved atomic.Int32
+ FromDataVerifyTimeSpentMS atomic.Int64
+ ResultCacheHits atomic.Int32
+ ResultCacheHitsWasted atomic.Int32
+ ResultCacheMisses atomic.Int32
+}
+
+var _ MetricsReporter = (*InMemoryMetrics)(nil)
+
+func (m *InMemoryMetrics) AddCredentialVerificationsSaved(count int) {
+ m.CredentialVerificationsSaved.Add(int32(count))
+}
+
+func (m *InMemoryMetrics) AddFromDataVerifyTimeSpent(wallTime time.Duration) {
+ m.FromDataVerifyTimeSpentMS.Add(wallTime.Milliseconds())
+}
+
+func (m *InMemoryMetrics) AddResultCacheHits(count int) {
+ m.ResultCacheHits.Add(int32(count))
+}
+
+func (m *InMemoryMetrics) AddResultCacheMisses(count int) {
+ m.ResultCacheMisses.Add(int32(count))
+}
+
+func (m *InMemoryMetrics) AddResultCacheHitsWasted(count int) {
+ m.ResultCacheHitsWasted.Add(int32(count))
+}
diff --git a/pkg/verificationcache/metrics_reporter.go b/pkg/verificationcache/metrics_reporter.go
new file mode 100644
index 000000000..c5f16865c
--- /dev/null
+++ b/pkg/verificationcache/metrics_reporter.go
@@ -0,0 +1,28 @@
+package verificationcache
+
+import "time"
+
+// MetricsReporter is an interface used by a verification cache to report various metrics related to its operation.
+// Implementations must be thread-safe.
+type MetricsReporter interface {
+ // AddCredentialVerificationsSaved records "saved" verification attempts, which is when credential verification
+ // status is loaded from the cache instead of retrieved from a remote verification endpoint. This number might be
+ // smaller than the cache hit count due to cache hit "wasting"; see AddResultCacheHitsWasted for more information.
+ AddCredentialVerificationsSaved(count int)
+
+ // AddFromDataVerifyTimeSpent records wall time spent in calls to detector.FromData with verify=true.
+ AddFromDataVerifyTimeSpent(wallTime time.Duration)
+
+ // AddResultCacheHits records result cache hits. Not all cache hits result in elided remote verification requests
+ // due to cache hit "wasting"; see AddResultCacheHitsWasted for more information.
+ AddResultCacheHits(count int)
+
+ // AddResultCacheMisses records result cache misses.
+ AddResultCacheMisses(count int)
+
+ // AddResultCacheHitsWasted records "wasted" result cache hits. A "wasted" result cache hit is a result cache hit
+ // that does not elide a remote verification request because there are other secret findings in the relevant chunk
+ // that are not cached. When this happens, the detector's FromData method must be called anyway, so the cache hit
+ // doesn't save any remote requests.
+ AddResultCacheHitsWasted(count int)
+}
diff --git a/pkg/verificationcache/result_cache.go b/pkg/verificationcache/result_cache.go
new file mode 100644
index 000000000..b915f65f3
--- /dev/null
+++ b/pkg/verificationcache/result_cache.go
@@ -0,0 +1,9 @@
+package verificationcache
+
+import (
+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+)
+
+// ResultCache is a cache that holds individual detector results. It serves as a component of a VerificationCache.
+type ResultCache cache.Cache[detectors.Result]
diff --git a/pkg/verificationcache/verification_cache.go b/pkg/verificationcache/verification_cache.go
new file mode 100644
index 000000000..8f2c9f609
--- /dev/null
+++ b/pkg/verificationcache/verification_cache.go
@@ -0,0 +1,139 @@
+package verificationcache
+
+import (
+ "sync"
+ "time"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/hasher"
+)
+
+// VerificationCache is a structure that can be used to cache verification results from detectors so that a given
+// credential does not trigger multiple identical remote verification attempts.
+type VerificationCache struct {
+ metrics MetricsReporter
+ resultCache ResultCache
+
+ hashMu sync.Mutex
+ hasher hasher.Hasher
+}
+
+// New creates a new verification cache with the provided result cache and metrics reporter. If resultCache is nil, the
+// verification cache will be a no-op passthrough, although it will still record relevant metrics to the provided
+// metrics reporter in this case.
+func New(resultCache ResultCache, metrics MetricsReporter) *VerificationCache {
+ if metrics == nil {
+ metrics = &InMemoryMetrics{}
+ }
+
+ return &VerificationCache{
+ metrics: metrics,
+ resultCache: resultCache,
+ hasher: hasher.NewBlake2B(),
+ }
+}
+
+// FromData is a cache-aware facade in front of the provided detector's FromData method.
+//
+// If the verification cache's underlying result cache is nil, or verify is false, or forceCacheUpdate is true, this
+// method invokes the provided detector's FromData method with the provided arguments and returns the result. If the
+// result cache is non-nil and forceCacheUpdate is true, the result cache is updated with the results before they are
+// returned.
+//
+// Otherwise, the detector's FromData method is called with verify=false. The result cache is then checked for each
+// returned result. If there is a cache hit for each result, these cached values are all returned. Otherwise, the
+// detector's FromData method is called again, but with verify=true, and the results are stored in the cache and then
+// returned.
+func (v *VerificationCache) FromData(
+ ctx context.Context,
+ detector detectors.Detector,
+ verify bool,
+ forceCacheUpdate bool,
+ data []byte,
+) ([]detectors.Result, error) {
+
+ if v.resultCache == nil {
+ if verify {
+ start := time.Now()
+ defer func() {
+ v.metrics.AddFromDataVerifyTimeSpent(time.Since(start))
+ }()
+ }
+
+ return detector.FromData(ctx, verify, data)
+ }
+
+ if !forceCacheUpdate {
+ withoutRemoteVerification, err := detector.FromData(ctx, false, data)
+ if err != nil {
+ return nil, err
+ }
+
+ if !verify {
+ return withoutRemoteVerification, nil
+ }
+
+ isEverythingCached := true
+ var cacheHitsInCurrentChunk int
+ for i, r := range withoutRemoteVerification {
+ cacheKey, err := v.getResultCacheKey(r)
+ if err != nil {
+ ctx.Logger().Error(err, "error getting result cache key for verification caching",
+ "operation", "read")
+ isEverythingCached = false
+ v.metrics.AddResultCacheHitsWasted(cacheHitsInCurrentChunk)
+ break
+ }
+ if cacheHit, ok := v.resultCache.Get(string(cacheKey)); ok {
+ withoutRemoteVerification[i].CopyVerificationInfo(&cacheHit)
+ withoutRemoteVerification[i].VerificationFromCache = true
+ v.metrics.AddResultCacheHits(1)
+ cacheHitsInCurrentChunk++
+ } else {
+ v.metrics.AddResultCacheMisses(1)
+ isEverythingCached = false
+ v.metrics.AddResultCacheHitsWasted(cacheHitsInCurrentChunk)
+ break
+ }
+ }
+
+ if isEverythingCached {
+ v.metrics.AddCredentialVerificationsSaved(len(withoutRemoteVerification))
+ return withoutRemoteVerification, nil
+ }
+ }
+
+ start := time.Now()
+ withRemoteVerification, err := detector.FromData(ctx, verify, data)
+ if verify {
+ v.metrics.AddFromDataVerifyTimeSpent(time.Since(start))
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ for _, r := range withRemoteVerification {
+ cacheKey, err := v.getResultCacheKey(r)
+ if err != nil {
+ ctx.Logger().Error(err, "error getting result cache key for verification caching",
+ "operation", "write")
+ continue
+ }
+
+ copyForCaching := r
+ // Do not persist raw secret values in a long-lived cache
+ copyForCaching.Raw = nil
+ copyForCaching.RawV2 = nil
+ v.resultCache.Set(string(cacheKey), copyForCaching)
+ }
+
+ return withRemoteVerification, nil
+}
+
+func (v *VerificationCache) getResultCacheKey(result detectors.Result) ([]byte, error) {
+ v.hashMu.Lock()
+ defer v.hashMu.Unlock()
+
+ return v.hasher.Hash(append(result.Raw, result.RawV2...))
+}
diff --git a/pkg/verificationcache/verification_cache_test.go b/pkg/verificationcache/verification_cache_test.go
new file mode 100644
index 000000000..69476844d
--- /dev/null
+++ b/pkg/verificationcache/verification_cache_test.go
@@ -0,0 +1,240 @@
+package verificationcache
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
+ logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type testDetector struct {
+ fromDataCallCount int
+ results []detectors.Result
+}
+
+func (t *testDetector) FromData(_ context.Context, verify bool, _ []byte) ([]detectors.Result, error) {
+ t.fromDataCallCount = t.fromDataCallCount + 1
+ var results []detectors.Result
+ for _, r := range t.results {
+ copy := detectors.Result{Redacted: r.Redacted, Raw: r.Raw, RawV2: r.RawV2}
+ if verify {
+ copy.CopyVerificationInfo(&r)
+ }
+ results = append(results, copy)
+ }
+
+ // The metric timing resolution is 1 ms, so the detector needs to artificially slow down so that it can actually be
+ // monitored.
+ time.Sleep(2 * time.Millisecond)
+
+ return results, nil
+}
+
+func (t *testDetector) Keywords() []string { return nil }
+func (t *testDetector) Type() detectorspb.DetectorType { return -1 }
+func (t *testDetector) Description() string { return "" }
+
+var _ detectors.Detector = (*testDetector)(nil)
+
+func getResultCacheKey(t *testing.T, cache *VerificationCache, result detectors.Result) string {
+ key, err := cache.getResultCacheKey(result)
+ require.NoError(t, err)
+ return string(key)
+}
+
+func TestVerificationCacheFromData_Passthrough(t *testing.T) {
+ detector := testDetector{results: []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},
+ }}
+
+ require.NotPanics(t, func() {
+ cache := New(nil, nil)
+ results, err := cache.FromData(
+ logContext.Background(),
+ &detector,
+ true,
+ true,
+ nil)
+
+ require.NoError(t, err)
+ assert.Equal(t, 1, detector.fromDataCallCount)
+ assert.ElementsMatch(t, detector.results, results)
+ })
+}
+
+func TestVerificationCacheFromData_VerifyFalseForceCacheUpdateFalse(t *testing.T) {
+ detector := testDetector{results: []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},
+ }}
+ metrics := InMemoryMetrics{}
+ cache := New(simple.NewCache[detectors.Result](), &metrics)
+
+ results, err := cache.FromData(
+ logContext.Background(),
+ &detector,
+ false,
+ false,
+ nil)
+
+ require.NoError(t, err)
+ assert.Equal(t, 1, detector.fromDataCallCount)
+ assert.ElementsMatch(t, []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: false},
+ }, results)
+ assert.Empty(t, cache.resultCache.Values())
+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())
+ assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheHits.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load())
+}
+
+func TestFromDataCached_VerifyFalseForceCacheUpdateTrue(t *testing.T) {
+ detector := testDetector{results: []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},
+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},
+ }}
+ detector.results[1].SetVerificationError(errors.New("test verification error"))
+ metrics := InMemoryMetrics{}
+ cache := New(simple.NewCache[detectors.Result](), &metrics)
+
+ results, err := cache.FromData(
+ logContext.Background(),
+ &detector,
+ false,
+ true,
+ nil)
+
+ require.NoError(t, err)
+ assert.Equal(t, 1, detector.fromDataCallCount)
+ assert.ElementsMatch(t, []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: false},
+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},
+ }, results)
+ assert.ElementsMatch(t, []detectors.Result{
+ {Redacted: "hello", Verified: false},
+ {Redacted: "world", Verified: false},
+ }, cache.resultCache.Values())
+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())
+ assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheHits.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load())
+}
+
+func TestFromDataCached_VerifyTrueForceCacheUpdateFalseAllCacheHits(t *testing.T) {
+ remoteResults := []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},
+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},
+ }
+ remoteResults[1].SetVerificationError(errors.New("test verification error"))
+ detector := testDetector{results: remoteResults}
+ cacheData := []detectors.Result{
+ {Redacted: "hello", Verified: false},
+ {Redacted: "world", Verified: true},
+ }
+ cacheData[0].SetVerificationError(errors.New("test verification error"))
+ metrics := InMemoryMetrics{}
+ cache := New(simple.NewCache[detectors.Result](), &metrics)
+ cache.resultCache.Set(getResultCacheKey(t, cache, remoteResults[0]), cacheData[0])
+ cache.resultCache.Set(getResultCacheKey(t, cache, remoteResults[1]), cacheData[1])
+
+ results, err := cache.FromData(
+ logContext.Background(),
+ &detector,
+ true,
+ false,
+ nil)
+
+ require.NoError(t, err)
+ assert.Equal(t, 1, detector.fromDataCallCount)
+ wantResults := []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: false, VerificationFromCache: true},
+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: true, VerificationFromCache: true},
+ }
+ wantResults[0].SetVerificationError(errors.New("test verification error"))
+ assert.ElementsMatch(t, wantResults, results)
+ assert.ElementsMatch(t, cacheData, cache.resultCache.Values())
+ assert.Equal(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())
+ assert.Equal(t, int32(2), metrics.CredentialVerificationsSaved.Load())
+ assert.Equal(t, int32(2), metrics.ResultCacheHits.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load())
+}
+
+func TestFromDataCached_VerifyTrueForceCacheUpdateFalseCacheMiss(t *testing.T) {
+ detector := testDetector{results: []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},
+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},
+ }}
+ detector.results[1].SetVerificationError(errors.New("test verification error"))
+ cachedResult := detectors.Result{Redacted: "hello", Verified: false}
+ cachedResult.SetVerificationError(errors.New("test verification error"))
+ resultCache := simple.NewCache[detectors.Result]()
+ metrics := InMemoryMetrics{}
+ cache := New(resultCache, &metrics)
+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), cachedResult)
+
+ results, err := cache.FromData(
+ logContext.Background(),
+ &detector,
+ true,
+ false,
+ nil)
+
+ require.NoError(t, err)
+ assert.Equal(t, 2, detector.fromDataCallCount)
+ assert.ElementsMatch(t, detector.results, results)
+ wantCacheData := []detectors.Result{
+ {Redacted: "hello", Verified: true},
+ {Redacted: "world", Verified: false},
+ }
+ wantCacheData[1].SetVerificationError(errors.New("test verification error"))
+ assert.ElementsMatch(t, wantCacheData, cache.resultCache.Values())
+ assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())
+ assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load())
+ assert.Equal(t, int32(1), metrics.ResultCacheHits.Load())
+ assert.Equal(t, int32(1), metrics.ResultCacheMisses.Load())
+ assert.Equal(t, int32(1), metrics.ResultCacheHitsWasted.Load())
+}
+
+func TestFromDataCached_VerifyTrueForceCacheUpdateTrue(t *testing.T) {
+ detector := testDetector{results: []detectors.Result{
+ {Redacted: "hello", Raw: []byte("hello"), RawV2: []byte("helloV2"), Verified: true},
+ {Redacted: "world", Raw: []byte("world"), RawV2: []byte("worldV2"), Verified: false},
+ }}
+ detector.results[1].SetVerificationError(errors.New("test verification error"))
+ metrics := InMemoryMetrics{}
+ cache := New(simple.NewCache[detectors.Result](), &metrics)
+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[0]), detectors.Result{Redacted: "hello", Verified: false})
+ cache.resultCache.Set(getResultCacheKey(t, cache, detector.results[1]), detectors.Result{Redacted: "world", Verified: true})
+
+ results, err := cache.FromData(
+ logContext.Background(),
+ &detector,
+ true,
+ true,
+ nil)
+
+ require.NoError(t, err)
+ assert.Equal(t, 1, detector.fromDataCallCount)
+ assert.ElementsMatch(t, detector.results, results)
+ wantCacheData := []detectors.Result{
+ {Redacted: "hello", Verified: true},
+ {Redacted: "world", Verified: false},
+ }
+ wantCacheData[1].SetVerificationError(errors.New("test verification error"))
+ assert.ElementsMatch(t, wantCacheData, cache.resultCache.Values())
+ assert.Less(t, int64(0), metrics.FromDataVerifyTimeSpentMS.Load())
+ assert.Equal(t, int32(0), metrics.CredentialVerificationsSaved.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheHits.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheMisses.Load())
+ assert.Equal(t, int32(0), metrics.ResultCacheHitsWasted.Load())
+}