Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Time detector_execution_duration per detector run (#5526)
Dave Hubbard6ced18b79b37
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex a40d1e7e6..a04e4b61e 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -1215,6 +1215,10 @@ func (e *Engine) detectorWorker(ctx context.Context) {}func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {+ // start feeds only the printAvgDetectorTime report, so it is left at its zero value when that option is off and+ // must not be read outside that report's code. It is taken once per chunk, so each duration the report records+ // spans every match processed so far. detectorExecutionDuration times each match on its own instead; see+ // matchStart below.var start time.Timeif e.printAvgDetectorTime {start = time.Now()@@ -1251,12 +1255,16 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {t := time.AfterFunc(detectionTimeout+1*time.Second, func() {ctx.Logger().Error(nil, "a detector ignored the context timeout")})+ // detectorExecutionDuration covers one detector run, which is this single FromData call, so it is timed here+ // regardless of printAvgDetectorTime.+ matchStart := time.Now()results, err := e.verificationCache.FromData(ctx,data.detector.Detector,data.verify,data.chunk.SecretID != 0,matchBytes)+ matchElapsed := time.Since(matchStart)t.Stop()cancel()if err != nil {@@ -1285,7 +1293,7 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {).Inc()detectorExecutionDuration.WithLabelValues(detectorNameStr,- ).Observe(float64(time.Since(start).Milliseconds()))+ ).Observe(float64(matchElapsed.Milliseconds()))if e.printAvgDetectorTime && len(results) > 0 {elapsed := time.Since(start)diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.goindex d7aeb9cb9..13e9260cb 100644--- a/pkg/engine/engine_test.go+++ b/pkg/engine/engine_test.go@@ -9,6 +9,7 @@ import ("net/http/httptest""os""path/filepath"+ "strings""sync""sync/atomic""testing"@@ -16,12 +17,14 @@ import ("github.com/google/go-cmp/cmp"lru "github.com/hashicorp/golang-lru/v2"+ "github.com/prometheus/client_golang/prometheus""github.com/stretchr/testify/assert""github.com/stretchr/testify/require""google.golang.org/protobuf/testing/protocmp""pgregory.net/rapid""github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/config""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/custom_detectors"@@ -2040,6 +2043,96 @@ func TestEngine_DetectChunk_CacheHitIsNotVerificationSkipped(t *testing.T) {}}+// slowDetectorType is used only by slowDetector, so its detectorExecutionDuration series isn't shared with other tests.+const slowDetectorType = detector_typepb.DetectorType(-1001)++// slowDetector takes a fixed time on every FromData call and returns no results, so tests can check what+// detectorExecutionDuration records for each detector run.+type slowDetector struct {+ delay time.Duration+}++func (d slowDetector) FromData(_ aCtx.Context, _ bool, _ []byte) ([]detectors.Result, error) {+ time.Sleep(d.delay)+ return nil, nil+}+func (d slowDetector) Keywords() []string { return []string{"keyword"} }+func (d slowDetector) Type() detector_typepb.DetectorType { return slowDetectorType }+func (d slowDetector) Description() string { return "slow detector for testing" }++// detectorExecutionDurationSnapshot reads one detector's detectorExecutionDuration series from the default gatherer.+// It returns the observation count and the cumulative count for each bucket upper bound.+func detectorExecutionDurationSnapshot(t *testing.T, detectorName string) (uint64, map[float64]uint64) {+ t.Helper()++ families, err := prometheus.DefaultGatherer.Gather()+ require.NoError(t, err)++ name := prometheus.BuildFQName(common.MetricsNamespace, common.MetricsSubsystem, "detector_execution_duration")+ buckets := map[float64]uint64{}+ for _, family := range families {+ if family.GetName() != name {+ continue+ }+ for _, metric := range family.GetMetric() {+ for _, label := range metric.GetLabel() {+ if label.GetName() != "detector_name" || label.GetValue() != detectorName {+ continue+ }+ for _, bucket := range metric.GetHistogram().GetBucket() {+ buckets[bucket.GetUpperBound()] = bucket.GetCumulativeCount()+ }+ return metric.GetHistogram().GetSampleCount(), buckets+ }+ }+ }+ return 0, buckets+}++// TestEngine_DetectChunk_RecordsDurationPerDetectorRun validates that detectorExecutionDuration records one+// observation per keyword match, each timing only that match's FromData call, whether or not the+// printAvgDetectorTime report is enabled.+func TestEngine_DetectChunk_RecordsDurationPerDetectorRun(t *testing.T) {+ ctx := context.Background()+ const delay = 80 * time.Millisecond++ for _, printAvgDetectorTime := range []bool{false, true} {+ t.Run(fmt.Sprintf("printAvgDetectorTime=%v", printAvgDetectorTime), func(t *testing.T) {+ // Arrange: Create a minimal engine with a passthrough verification cache.+ e := &Engine{+ verificationCache: verificationcache.New(nil, &verificationcache.InMemoryMetrics{}),+ printAvgDetectorTime: printAvgDetectorTime,+ }++ // Arrange: Place two keywords far enough apart that their match windows don't merge, so the detector runs+ // twice on one chunk.+ data := []byte("keyword" + strings.Repeat(" ", 4096) + "keyword")+ ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{slowDetector{delay: delay}})+ detectorMatches := ahcore.FindDetectorMatches(data)+ require.Len(t, detectorMatches, 1)+ require.Len(t, detectorMatches[0].Matches(), 2)++ detectorName := slowDetectorType.String()+ countBefore, bucketsBefore := detectorExecutionDurationSnapshot(t, detectorName)++ // Act+ e.detectChunk(ctx, detectableChunk{+ detector: detectorMatches[0],+ verify: true,+ wgDoneFn: func() {},+ })++ // Assert: Each run takes about 80ms, so per-run timing puts both observations above the 25ms bound and at+ // or below the 125ms bound. Timing from the start of the chunk would put the second run near 160ms, and+ // an unset start time would put both past the largest bound.+ countAfter, bucketsAfter := detectorExecutionDurationSnapshot(t, detectorName)+ assert.Equal(t, uint64(2), countAfter-countBefore)+ assert.Equal(t, uint64(0), bucketsAfter[25]-bucketsBefore[25])+ assert.Equal(t, uint64(2), bucketsAfter[125]-bucketsBefore[125])+ })+ }+}+// TestEngine_ScannerWorker_DetectableChunkHasCorrectVerifyFlag validates that scannerWorker generates detectableChunk// structs that have the correct verify flag set. It also validates that the original chunks' SourceVerify flags are// unchanged.diff --git a/pkg/engine/metrics.go b/pkg/engine/metrics.goindex b929c4f01..6c3ce4ece 100644--- a/pkg/engine/metrics.go+++ b/pkg/engine/metrics.go@@ -19,7 +19,7 @@ var ([]string{"detector_name", "source_type", "source_name"},)- // Note this is the time taken to call FromData on each detector, not necessarily the time taken+ // Note this is the time taken by one FromData call, made once per keyword match, not necessarily the time taken// to verify a credential via an API call. If the regex match within FromData does not match, the// detector will return early. For now this is a good proxy for the time taken to verify a credential.// TODO (ahrav)