Skip to content

Find, verify, and analyze leaked credentials

Time detector_execution_duration per detector run (#5526)

Dave Hubbard6ced18b79b37
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index a40d1e7e6..a04e4b61e 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -1215,6 +1215,10 @@ func (e *Engine) detectorWorker(ctx context.Context) {
}
func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
+ // start feeds only the printAvgDetectorTime report, so it is left at its zero value when that option is off and
+ // must not be read outside that report's code. It is taken once per chunk, so each duration the report records
+ // spans every match processed so far. detectorExecutionDuration times each match on its own instead; see
+ // matchStart below.
var start time.Time
if e.printAvgDetectorTime {
start = time.Now()
@@ -1251,12 +1255,16 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
t := time.AfterFunc(detectionTimeout+1*time.Second, func() {
ctx.Logger().Error(nil, "a detector ignored the context timeout")
})
+ // detectorExecutionDuration covers one detector run, which is this single FromData call, so it is timed here
+ // regardless of printAvgDetectorTime.
+ matchStart := time.Now()
results, err := e.verificationCache.FromData(
ctx,
data.detector.Detector,
data.verify,
data.chunk.SecretID != 0,
matchBytes)
+ matchElapsed := time.Since(matchStart)
t.Stop()
cancel()
if err != nil {
@@ -1285,7 +1293,7 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {
).Inc()
detectorExecutionDuration.WithLabelValues(
detectorNameStr,
- ).Observe(float64(time.Since(start).Milliseconds()))
+ ).Observe(float64(matchElapsed.Milliseconds()))
if e.printAvgDetectorTime && len(results) > 0 {
elapsed := time.Since(start)
diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go
index d7aeb9cb9..13e9260cb 100644
--- a/pkg/engine/engine_test.go
+++ b/pkg/engine/engine_test.go
@@ -9,6 +9,7 @@ import (
"net/http/httptest"
"os"
"path/filepath"
+ "strings"
"sync"
"sync/atomic"
"testing"
@@ -16,12 +17,14 @@ import (
"github.com/google/go-cmp/cmp"
lru "github.com/hashicorp/golang-lru/v2"
+ "github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/testing/protocmp"
"pgregory.net/rapid"
"github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/custom_detectors"
@@ -2040,6 +2043,96 @@ func TestEngine_DetectChunk_CacheHitIsNotVerificationSkipped(t *testing.T) {
}
}
+// slowDetectorType is used only by slowDetector, so its detectorExecutionDuration series isn't shared with other tests.
+const slowDetectorType = detector_typepb.DetectorType(-1001)
+
+// slowDetector takes a fixed time on every FromData call and returns no results, so tests can check what
+// detectorExecutionDuration records for each detector run.
+type slowDetector struct {
+ delay time.Duration
+}
+
+func (d slowDetector) FromData(_ aCtx.Context, _ bool, _ []byte) ([]detectors.Result, error) {
+ time.Sleep(d.delay)
+ return nil, nil
+}
+func (d slowDetector) Keywords() []string { return []string{"keyword"} }
+func (d slowDetector) Type() detector_typepb.DetectorType { return slowDetectorType }
+func (d slowDetector) Description() string { return "slow detector for testing" }
+
+// detectorExecutionDurationSnapshot reads one detector's detectorExecutionDuration series from the default gatherer.
+// It returns the observation count and the cumulative count for each bucket upper bound.
+func detectorExecutionDurationSnapshot(t *testing.T, detectorName string) (uint64, map[float64]uint64) {
+ t.Helper()
+
+ families, err := prometheus.DefaultGatherer.Gather()
+ require.NoError(t, err)
+
+ name := prometheus.BuildFQName(common.MetricsNamespace, common.MetricsSubsystem, "detector_execution_duration")
+ buckets := map[float64]uint64{}
+ for _, family := range families {
+ if family.GetName() != name {
+ continue
+ }
+ for _, metric := range family.GetMetric() {
+ for _, label := range metric.GetLabel() {
+ if label.GetName() != "detector_name" || label.GetValue() != detectorName {
+ continue
+ }
+ for _, bucket := range metric.GetHistogram().GetBucket() {
+ buckets[bucket.GetUpperBound()] = bucket.GetCumulativeCount()
+ }
+ return metric.GetHistogram().GetSampleCount(), buckets
+ }
+ }
+ }
+ return 0, buckets
+}
+
+// TestEngine_DetectChunk_RecordsDurationPerDetectorRun validates that detectorExecutionDuration records one
+// observation per keyword match, each timing only that match's FromData call, whether or not the
+// printAvgDetectorTime report is enabled.
+func TestEngine_DetectChunk_RecordsDurationPerDetectorRun(t *testing.T) {
+ ctx := context.Background()
+ const delay = 80 * time.Millisecond
+
+ for _, printAvgDetectorTime := range []bool{false, true} {
+ t.Run(fmt.Sprintf("printAvgDetectorTime=%v", printAvgDetectorTime), func(t *testing.T) {
+ // Arrange: Create a minimal engine with a passthrough verification cache.
+ e := &Engine{
+ verificationCache: verificationcache.New(nil, &verificationcache.InMemoryMetrics{}),
+ printAvgDetectorTime: printAvgDetectorTime,
+ }
+
+ // Arrange: Place two keywords far enough apart that their match windows don't merge, so the detector runs
+ // twice on one chunk.
+ data := []byte("keyword" + strings.Repeat(" ", 4096) + "keyword")
+ ahcore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{slowDetector{delay: delay}})
+ detectorMatches := ahcore.FindDetectorMatches(data)
+ require.Len(t, detectorMatches, 1)
+ require.Len(t, detectorMatches[0].Matches(), 2)
+
+ detectorName := slowDetectorType.String()
+ countBefore, bucketsBefore := detectorExecutionDurationSnapshot(t, detectorName)
+
+ // Act
+ e.detectChunk(ctx, detectableChunk{
+ detector: detectorMatches[0],
+ verify: true,
+ wgDoneFn: func() {},
+ })
+
+ // Assert: Each run takes about 80ms, so per-run timing puts both observations above the 25ms bound and at
+ // or below the 125ms bound. Timing from the start of the chunk would put the second run near 160ms, and
+ // an unset start time would put both past the largest bound.
+ countAfter, bucketsAfter := detectorExecutionDurationSnapshot(t, detectorName)
+ assert.Equal(t, uint64(2), countAfter-countBefore)
+ assert.Equal(t, uint64(0), bucketsAfter[25]-bucketsBefore[25])
+ assert.Equal(t, uint64(2), bucketsAfter[125]-bucketsBefore[125])
+ })
+ }
+}
+
// TestEngine_ScannerWorker_DetectableChunkHasCorrectVerifyFlag validates that scannerWorker generates detectableChunk
// structs that have the correct verify flag set. It also validates that the original chunks' SourceVerify flags are
// unchanged.
diff --git a/pkg/engine/metrics.go b/pkg/engine/metrics.go
index b929c4f01..6c3ce4ece 100644
--- a/pkg/engine/metrics.go
+++ b/pkg/engine/metrics.go
@@ -19,7 +19,7 @@ var (
[]string{"detector_name", "source_type", "source_name"},
)
- // Note this is the time taken to call FromData on each detector, not necessarily the time taken
+ // Note this is the time taken by one FromData call, made once per keyword match, not necessarily the time taken
// to verify a credential via an API call. If the regex match within FromData does not match, the
// detector will return early. For now this is a good proxy for the time taken to verify a credential.
// TODO (ahrav)