Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Make worker multipliers configurable (#3267)
Dustin Deckerdb0108f73179
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex dfc5f4d88..909dfa9fd 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -135,6 +135,15 @@ type Config struct {// that have been detected by multiple detectors.// By default, it is set to true.VerificationOverlap bool++ // DetectorWorkerMultiplier is used to determine the number of detector workers to spawn.+ DetectorWorkerMultiplier int++ // NotificationWorkerMultiplier is used to determine the number of notification workers to spawn.+ NotificationWorkerMultiplier int++ // VerificationOverlapWorkerMultiplier is used to determine the number of verification overlap workers to spawn.+ VerificationOverlapWorkerMultiplier int}// Engine represents the core scanning engine responsible for detecting secrets in input data.@@ -195,24 +204,34 @@ type Engine struct {// Note: bad hack only used for testing.verificationOverlapTracker *verificationOverlapTracker++ // detectorWorkerMultiplier is used to calculate the number of detector workers.+ detectorWorkerMultiplier int+ // notificationWorkerMultiplier is used to calculate the number of notification workers.+ notificationWorkerMultiplier int+ // verificationOverlapWorkerMultiplier is used to calculate the number of verification overlap workers.+ verificationOverlapWorkerMultiplier int}// NewEngine creates a new Engine instance with the provided configuration.func NewEngine(ctx context.Context, cfg *Config) (*Engine, error) {engine := &Engine{- concurrency: cfg.Concurrency,- decoders: cfg.Decoders,- detectors: cfg.Detectors,- dispatcher: cfg.Dispatcher,- verify: cfg.Verify,- filterUnverified: cfg.FilterUnverified,- filterEntropy: cfg.FilterEntropy,- printAvgDetectorTime: cfg.PrintAvgDetectorTime,- retainFalsePositives: cfg.LogFilteredUnverified,- verificationOverlap: cfg.VerificationOverlap,- sourceManager: cfg.SourceManager,- scanEntireChunk: cfg.ShouldScanEntireChunk,- detectorVerificationOverrides: cfg.DetectorVerificationOverrides,+ concurrency: cfg.Concurrency,+ decoders: cfg.Decoders,+ detectors: cfg.Detectors,+ dispatcher: cfg.Dispatcher,+ verify: cfg.Verify,+ filterUnverified: cfg.FilterUnverified,+ filterEntropy: cfg.FilterEntropy,+ printAvgDetectorTime: cfg.PrintAvgDetectorTime,+ retainFalsePositives: cfg.LogFilteredUnverified,+ verificationOverlap: cfg.VerificationOverlap,+ sourceManager: cfg.SourceManager,+ scanEntireChunk: cfg.ShouldScanEntireChunk,+ detectorVerificationOverrides: cfg.DetectorVerificationOverrides,+ detectorWorkerMultiplier: cfg.DetectorWorkerMultiplier,+ notificationWorkerMultiplier: cfg.NotificationWorkerMultiplier,+ verificationOverlapWorkerMultiplier: cfg.VerificationOverlapWorkerMultiplier,}if engine.sourceManager == nil {return nil, fmt.Errorf("source manager is required")@@ -303,7 +322,19 @@ func (e *Engine) setDefaults(ctx context.Context) {ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)e.concurrency = numCPU}- ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)++ if e.detectorWorkerMultiplier < 1 {+ // bound by net i/o so it's higher than other workers+ e.detectorWorkerMultiplier = 8+ }++ if e.notificationWorkerMultiplier < 1 {+ e.notificationWorkerMultiplier = 1+ }++ if e.verificationOverlapWorkerMultiplier < 1 {+ e.verificationOverlapWorkerMultiplier = 1+ }// Default decoders handle common encoding formats.if len(e.decoders) == 0 {@@ -625,9 +656,10 @@ func (e *Engine) startScannerWorkers(ctx context.Context) {}func (e *Engine) startDetectorWorkers(ctx context.Context) {- const detectorWorkerMultiplier = 4- ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)- for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {+ numWorkers := e.concurrency * e.detectorWorkerMultiplier++ ctx.Logger().V(2).Info("starting detector workers", "count", numWorkers)+ for worker := 0; worker < numWorkers; worker++ {e.wgDetectorWorkers.Add(1)go func() {ctx := context.WithValue(ctx, "detector_worker_id", common.RandomID(5))@@ -639,8 +671,10 @@ func (e *Engine) startDetectorWorkers(ctx context.Context) {}func (e *Engine) startVerificationOverlapWorkers(ctx context.Context) {- ctx.Logger().V(2).Info("starting verificationOverlap workers", "count", e.concurrency)- for worker := uint64(0); worker < uint64(e.concurrency); worker++ {+ numWorkers := e.concurrency * e.verificationOverlapWorkerMultiplier++ ctx.Logger().V(2).Info("starting verificationOverlap workers", "count", numWorkers)+ for worker := 0; worker < numWorkers; worker++ {e.verificationOverlapWg.Add(1)go func() {ctx := context.WithValue(ctx, "verification_overlap_worker_id", common.RandomID(5))@@ -652,13 +686,10 @@ func (e *Engine) startVerificationOverlapWorkers(ctx context.Context) {}func (e *Engine) startNotifierWorkers(ctx context.Context) {- const notifierWorkerRatio = 2- maxNotifierWorkers := 1- if numWorkers := e.concurrency / notifierWorkerRatio; numWorkers > 0 {- maxNotifierWorkers = numWorkers- }- ctx.Logger().V(2).Info("starting notifier workers", "count", maxNotifierWorkers)- for worker := 0; worker < maxNotifierWorkers; worker++ {+ numWorkers := e.notificationWorkerMultiplier * e.concurrency++ ctx.Logger().V(2).Info("starting notifier workers", "count", numWorkers)+ for worker := 0; worker < numWorkers; worker++ {e.WgNotifier.Add(1)go func() {ctx := context.WithValue(ctx, "notifier_worker_id", common.RandomID(5))