Skip to content

Find, verify, and analyze leaked credentials

Filter unique detectors by keywords in chunk (#1711)

ahrav68f28a0e34f9
diff --git a/pkg/decoders/base64.go b/pkg/decoders/base64.go
index 6aa1f2c93..56b46409e 100644
--- a/pkg/decoders/base64.go
+++ b/pkg/decoders/base64.go
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/base64"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -25,7 +26,8 @@ func init() {
}
}
-func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
+func (d *Base64) FromChunk(chunk *sources.Chunk) *DecodableChunk {
+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_BASE64}
encodedSubstrings := getSubstringsOfCharacterSet(chunk.Data, 20, b64CharsetMapping, b64EndChars)
decodedSubstrings := make(map[string][]byte)
@@ -61,7 +63,7 @@ func (d *Base64) FromChunk(chunk *sources.Chunk) *sources.Chunk {
}
result.Write(chunk.Data[start:])
chunk.Data = result.Bytes()
- return chunk
+ return decodableChunk
}
return nil
diff --git a/pkg/decoders/decoders.go b/pkg/decoders/decoders.go
index 52b4709e0..b5e476899 100644
--- a/pkg/decoders/decoders.go
+++ b/pkg/decoders/decoders.go
@@ -1,6 +1,7 @@
package decoders
import (
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
@@ -13,8 +14,15 @@ func DefaultDecoders() []Decoder {
}
}
+// DecodableChunk is a chunk that includes the type of decoder used.
+// This allows us to avoid a type assertion on each decoder.
+type DecodableChunk struct {
+ *sources.Chunk
+ DecoderType detectorspb.DecoderType
+}
+
type Decoder interface {
- FromChunk(chunk *sources.Chunk) *sources.Chunk
+ FromChunk(chunk *sources.Chunk) *DecodableChunk
}
// Fuzz is an entrypoint for go-fuzz, which is an AFL-style fuzzing tool.
diff --git a/pkg/decoders/utf16.go b/pkg/decoders/utf16.go
index d5599a520..c534b5812 100644
--- a/pkg/decoders/utf16.go
+++ b/pkg/decoders/utf16.go
@@ -5,22 +5,24 @@ import (
"encoding/binary"
"unicode/utf8"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type UTF16 struct{}
-func (d *UTF16) FromChunk(chunk *sources.Chunk) *sources.Chunk {
+func (d *UTF16) FromChunk(chunk *sources.Chunk) *DecodableChunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_UTF16}
if utf16Data, err := utf16ToUTF8(chunk.Data); err == nil {
if len(utf16Data) == 0 {
return nil
}
chunk.Data = utf16Data
- return chunk
+ return decodableChunk
}
return nil
diff --git a/pkg/decoders/utf8.go b/pkg/decoders/utf8.go
index 93090e71f..be4c22dfa 100644
--- a/pkg/decoders/utf8.go
+++ b/pkg/decoders/utf8.go
@@ -4,22 +4,25 @@ import (
"bytes"
"unicode/utf8"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
)
type UTF8 struct{}
-func (d *UTF8) FromChunk(chunk *sources.Chunk) *sources.Chunk {
+func (d *UTF8) FromChunk(chunk *sources.Chunk) *DecodableChunk {
if chunk == nil || len(chunk.Data) == 0 {
return nil
}
+ decodableChunk := &DecodableChunk{Chunk: chunk, DecoderType: detectorspb.DecoderType_PLAIN}
+
if !utf8.Valid(chunk.Data) {
chunk.Data = extractSubstrings(chunk.Data)
- return chunk
+ return decodableChunk
}
- return chunk
+ return decodableChunk
}
// extractSubstrings performs similarly to the strings binutil,
diff --git a/pkg/engine/ahocorasickcore.go b/pkg/engine/ahocorasickcore.go
new file mode 100644
index 000000000..c5f8933d1
--- /dev/null
+++ b/pkg/engine/ahocorasickcore.go
@@ -0,0 +1,111 @@
+package engine
+
+import (
+ "strings"
+
+ ahocorasick "github.com/BobuSumisu/aho-corasick"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+// detectorKey is used to identify a detector in the keywordsToDetectors map.
+// Multiple detectors can have the same detector type but different versions.
+// This allows us to identify a detector by its type and version.
+type detectorKey struct {
+ detectorType detectorspb.DetectorType
+ version int
+}
+
+// detectorInfo is used to store a detector and whether it should be verified.
+type detectorInfo struct {
+ detectors.Detector
+ shouldVerify bool
+}
+
+// AhoCorasickCore encapsulates the operations and data structures used for keyword matching via the
+// Aho-Corasick algorithm. It is responsible for constructing and managing the trie for efficient
+// substring searches, as well as mapping keywords to their associated detectors for rapid lookups.
+type AhoCorasickCore struct {
+ // prefilter is a ahocorasick struct used for doing efficient string
+ // matching given a set of words. (keywords from the rules in the config)
+ prefilter ahocorasick.Trie
+ // Maps for efficient lookups during detection.
+ detectorTypeToDetectorInfo map[detectorKey]detectorInfo
+ detectors map[bool][]detectors.Detector
+ keywordsToDetectors map[string][]detectorKey
+}
+
+// NewAhoCorasickCore allocates and initializes a new instance of AhoCorasickCore.
+// It creates an empty keyword-to-detectors map for future string matching operations.
+// The map detectorTypeToDetectorInfo is pre-allocated based on the size of detectors
+// provided, for efficient storage and lookup of detector information.
+func NewAhoCorasickCore(detectors map[bool][]detectors.Detector) *AhoCorasickCore {
+ return &AhoCorasickCore{
+ keywordsToDetectors: make(map[string][]detectorKey),
+ detectors: detectors,
+ detectorTypeToDetectorInfo: make(map[detectorKey]detectorInfo, len(detectors[true])+len(detectors[false])),
+ }
+}
+
+// Setup initializes the internal state of AhoCorasickCore to prepare it for keyword matching.
+// This involves pre-filtering setup and lookup optimization, critical for the engine's performance.
+func (ac *AhoCorasickCore) Setup(ctx context.Context) {
+ // Prepare maps for fast detector lookups, instead of scanning through an array of detectors for every chunk.
+ var keywords []string
+ for verify, detectorsSet := range ac.detectors {
+ for _, d := range detectorsSet {
+ key := createDetectorKey(d)
+ ac.detectorTypeToDetectorInfo[key] = detectorInfo{Detector: d, shouldVerify: verify}
+ keywords = ac.extractAndMapKeywords(d, key, keywords)
+ }
+ }
+
+ // Use the Ahocorasick algorithm to create a trie structure for efficient keyword matching.
+ // This ensures that we can rapidly match against a vast set of keywords without individually comparing each one.
+ ac.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
+ ctx.Logger().V(4).Info("AhoCorasickCore Setup complete")
+}
+
+// createDetectorKey creates a unique key for each detector. This key based on type and version,
+// it ensures faster lookups and reduces redundancy in our main detector store.
+func createDetectorKey(d detectors.Detector) detectorKey {
+ detectorType := d.Type()
+ var version int
+ if v, ok := d.(detectors.Versioner); ok {
+ version = v.Version()
+ }
+ return detectorKey{detectorType: detectorType, version: version}
+}
+
+// extractAndMapKeywords captures keywords associated with each detector and maps them.
+// This allows us to quickly determine which detectors are relevant based on the presence of certain keywords.
+func (ac *AhoCorasickCore) extractAndMapKeywords(d detectors.Detector, key detectorKey, keywords []string) []string {
+ for _, kw := range d.Keywords() {
+ kwLower := strings.ToLower(kw)
+ keywords = append(keywords, kwLower)
+ ac.keywordsToDetectors[kwLower] = append(ac.keywordsToDetectors[kwLower], key)
+ }
+ return keywords
+}
+
+// MatchString performs a string match using the Aho-Corasick algorithm, returning an array of matches.
+// Designed for internal use within the AhoCorasickCore component.
+func (ac *AhoCorasickCore) MatchString(input string) []*ahocorasick.Match {
+ return ac.prefilter.MatchString(strings.ToLower(input))
+}
+
+// PopulateDetectorsByMatch populates the given detectorMap based on the Aho-Corasick match results.
+// This method is designed to reuse the same map for performance optimization,
+// reducing the need for repeated allocations within each detector worker in the engine.
+func (ac *AhoCorasickCore) PopulateDetectorsByMatch(match *ahocorasick.Match, detectors map[detectorspb.DetectorType]detectorInfo) bool {
+ matchedKeys, ok := ac.keywordsToDetectors[match.MatchString()]
+ if !ok {
+ return false
+ }
+ for _, key := range matchedKeys {
+ detectors[key.detectorType] = ac.detectorTypeToDetectorInfo[key]
+ }
+ return true
+}
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 83642f9f6..6159046c8 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -3,14 +3,11 @@ package engine
import (
"bytes"
"fmt"
- "reflect"
"runtime"
- "strings"
"sync"
"sync/atomic"
"time"
- ahocorasick "github.com/BobuSumisu/aho-corasick"
lru "github.com/hashicorp/golang-lru"
"google.golang.org/protobuf/proto"
@@ -66,9 +63,8 @@ type Engine struct {
onlyVerified bool
printAvgDetectorTime bool
- // prefilter is a ahocorasick struct used for doing efficient string
- // matching given a set of words (keywords from the rules in the config)
- prefilter ahocorasick.Trie
+ // ahoCorasickHandler manages the Aho-Corasick trie and related keyword lookups.
+ ahoCorasickCore *AhoCorasickCore
// Engine synchronization primitives.
sourceManager *sources.SourceManager
@@ -93,9 +89,10 @@ type Engine struct {
dedupeCache *lru.Cache
}
-type EngineOption func(*Engine)
+// Option is used to configure the engine during initialization using functional options.
+type Option func(*Engine)
-func WithConcurrency(concurrency uint8) EngineOption {
+func WithConcurrency(concurrency uint8) Option {
return func(e *Engine) {
e.concurrency = concurrency
}
@@ -103,7 +100,7 @@ func WithConcurrency(concurrency uint8) EngineOption {
const ignoreTag = "trufflehog:ignore"
-func WithDetectors(verify bool, d ...detectors.Detector) EngineOption {
+func WithDetectors(verify bool, d ...detectors.Detector) Option {
return func(e *Engine) {
if e.detectors == nil {
e.detectors = make(map[bool][]detectors.Detector)
@@ -116,7 +113,7 @@ func WithDetectors(verify bool, d ...detectors.Detector) EngineOption {
}
}
-func WithDecoders(decoders ...decoders.Decoder) EngineOption {
+func WithDecoders(decoders ...decoders.Decoder) Option {
return func(e *Engine) {
e.decoders = decoders
}
@@ -124,14 +121,14 @@ func WithDecoders(decoders ...decoders.Decoder) EngineOption {
// WithFilterUnverified sets the filterUnverified flag on the engine. If set to
// true, the engine will only return the first unverified result for a chunk for a detector.
-func WithFilterUnverified(filter bool) EngineOption {
+func WithFilterUnverified(filter bool) Option {
return func(e *Engine) {
e.filterUnverified = filter
}
}
// WithFilterEntropy filters out unverified results using Shannon entropy.
-func WithFilterEntropy(entropy float64) EngineOption {
+func WithFilterEntropy(entropy float64) Option {
return func(e *Engine) {
if entropy > 0 {
e.filterEntropy = &entropy
@@ -141,7 +138,7 @@ func WithFilterEntropy(entropy float64) EngineOption {
// WithOnlyVerified sets the onlyVerified flag on the engine. If set to true,
// the engine will only print verified results.
-func WithOnlyVerified(onlyVerified bool) EngineOption {
+func WithOnlyVerified(onlyVerified bool) Option {
return func(e *Engine) {
e.onlyVerified = onlyVerified
}
@@ -153,7 +150,7 @@ func WithOnlyVerified(onlyVerified bool) EngineOption {
// the engine is configured to print the results.
// Calculating the average time taken by each detector is an expensive operation
// and should be avoided unless specified by the user.
-func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
+func WithPrintAvgDetectorTime(printAvgDetectorTime bool) Option {
return func(e *Engine) {
e.printAvgDetectorTime = printAvgDetectorTime
}
@@ -163,7 +160,7 @@ func WithPrintAvgDetectorTime(printAvgDetectorTime bool) EngineOption {
// the filterFunc returns true, the detector will be included for scanning.
// This option applies to the existing list of detectors configured, so the
// order this option appears matters. All filtering happens before scanning.
-func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
+func WithFilterDetectors(filterFunc func(detectors.Detector) bool) Option {
return func(e *Engine) {
// If no detectors are configured, do nothing.
if e.detectors == nil {
@@ -175,7 +172,7 @@ func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption
}
// WithPrinter sets the Printer on the engine.
-func WithPrinter(printer Printer) EngineOption {
+func WithPrinter(printer Printer) Option {
return func(e *Engine) {
e.printer = printer
}
@@ -269,32 +266,60 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {
return avgTime
}
-// Start the engine with options.
-func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
- const (
- defaultChannelBuffer = 1
- // TODO (ahrav): Determine the optimal cache size.
- cacheSize = 512 // number of entries in the LRU cache
- )
+// Start initializes and activates the engine's processing pipeline.
+// It sets up various default configurations, prepares lookup structures for
+// detectors, conducts basic sanity checks, and kickstarts all necessary workers.
+// Once started, the engine begins processing input data to identify secrets.
+func Start(ctx context.Context, options ...Option) (*Engine, error) {
+ e := &Engine{}
+
+ if err := e.initialize(ctx, options...); err != nil {
+ return nil, err
+ }
+ e.setDefaults(ctx)
+ ctx.Logger().V(4).Info("setting up aho-corasick core")
+ e.ahoCorasickCore.Setup(ctx)
+ e.sanityChecks(ctx)
+ e.startWorkers(ctx)
+
+ return e, nil
+}
+
+const defaultChannelBuffer = 1
+
+// initialize prepares the engine's internal structures. The LRU cache optimizes
+// deduplication efforts, allowing the engine to quickly check if a chunk has
+// been processed before, thereby saving computational overhead.
+func (e *Engine) initialize(ctx context.Context, options ...Option) error {
+ // TODO (ahrav): Determine the optimal cache size.
+ const cacheSize = 512 // number of entries in the LRU cache
cache, err := lru.New(cacheSize)
if err != nil {
- return nil, fmt.Errorf("failed to initialize LRU cache: %w", err)
+ return fmt.Errorf("failed to initialize LRU cache: %w", err)
}
- e := &Engine{
- detectableChunksChan: make(chan detectableChunk, defaultChannelBuffer),
- results: make(chan detectors.ResultWithMetadata, defaultChannelBuffer),
- dedupeCache: cache,
- printer: new(output.PlainPrinter), // default printer
- metrics: runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}},
- }
+ // Channels are used for communication between different parts of the engine,
+ // ensuring that data flows smoothly without race conditions.
+ e.detectableChunksChan = make(chan detectableChunk, defaultChannelBuffer)
+ e.results = make(chan detectors.ResultWithMetadata, defaultChannelBuffer)
+ e.dedupeCache = cache
+ e.printer = new(output.PlainPrinter)
+ e.metrics = runtimeMetrics{Metrics: Metrics{scanStartTime: time.Now()}}
for _, option := range options {
option(e)
}
+ ctx.Logger().V(4).Info("engine initialized")
+ e.ahoCorasickCore = NewAhoCorasickCore(e.detectors)
- // Set defaults.
+ return nil
+}
+
+// setDefaults ensures that if specific engine properties aren't provided,
+// they're set to reasonable default values. It makes the engine robust to
+// incomplete configuration.
+func (e *Engine) setDefaults(ctx context.Context) {
if e.concurrency == 0 {
numCPU := runtime.NumCPU()
ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)
@@ -302,12 +327,13 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
}
ctx.Logger().V(3).Info("engine started", "workers", e.concurrency)
- // Create SourceManager.
e.sourceManager = sources.NewManager(
sources.WithConcurrentSources(int(e.concurrency)),
sources.WithConcurrentUnits(int(e.concurrency)),
+ sources.WithBufferedOutput(defaultChannelBuffer),
)
+ // Default decoders handle common encoding formats.
if len(e.decoders) == 0 {
e.decoders = decoders.DefaultDecoders()
}
@@ -317,46 +343,30 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
e.detectors[true] = DefaultDetectors()
e.detectors[false] = []detectors.Detector{}
}
+ ctx.Logger().V(4).Info("default engine options set")
+}
- // build ahocorasick prefilter for efficient string matching
- // on keywords
- keywords := []string{}
- for _, d := range e.detectors[false] {
- for _, kw := range d.Keywords() {
- keywords = append(keywords, strings.ToLower(kw))
- }
- }
- for _, d := range e.detectors[true] {
- for _, kw := range d.Keywords() {
- keywords = append(keywords, strings.ToLower(kw))
- }
- }
- e.prefilter = *ahocorasick.NewTrieBuilder().AddStrings(keywords).Build()
-
- ctx.Logger().V(3).Info("loaded decoders", "count", len(e.decoders))
- ctx.Logger().V(3).Info("loaded detectors",
- "total", len(e.detectors[true])+len(e.detectors[false]),
- "verification_enabled", len(e.detectors[true]),
- "verification_disabled", len(e.detectors[false]),
- )
-
- // Sanity check detectors for duplicate configuration. Only log in case
- // a detector has been configured in a way that isn't represented by
- // the DetectorID (type and version).
- {
- dets := append(e.detectors[true], e.detectors[false]...)
- seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
- for _, det := range dets {
- id := config.GetDetectorID(det)
- if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
- ctx.Logger().Info("possible duplicate detector configured", "detector", id)
- }
- seenDetectors[id] = struct{}{}
+// Sanity check detectors for duplicate configuration. Only log in case
+// a detector has been configured in a way that isn't represented by
+// the DetectorID (type and version).
+func (e *Engine) sanityChecks(ctx context.Context) {
+ dets := append(e.detectors[true], e.detectors[false]...)
+ seenDetectors := make(map[config.DetectorID]struct{}, len(dets))
+ for _, det := range dets {
+ id := config.GetDetectorID(det)
+ if _, ok := seenDetectors[id]; ok && id.ID != detectorspb.DetectorType_CustomRegex {
+ ctx.Logger().Info("possible duplicate detector configured", "detector", id)
}
+ seenDetectors[id] = struct{}{}
}
+}
+// startWorkers initiates all necessary workers. Workers handle processing of
+// chunks concurrently. Separating the initialization of different types of
+// workers helps in scalability and makes it easier to diagnose issues.
+func (e *Engine) startWorkers(ctx context.Context) {
+ // Scanner workers process input data and extract chunks for detectors.
ctx.Logger().V(2).Info("starting scanner workers", "count", e.concurrency)
- // Run the Secret scanner workers and Notifier pipelines.
for worker := uint64(0); worker < uint64(e.concurrency); worker++ {
e.workersWg.Add(1)
go func() {
@@ -367,6 +377,7 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
}()
}
+ // Detector workers apply keyword matching, regexes and API calls to detect secrets in chunks.
const detectorWorkerMultiplier = 50
ctx.Logger().V(2).Info("starting detector workers", "count", e.concurrency*detectorWorkerMultiplier)
for worker := uint64(0); worker < uint64(e.concurrency*detectorWorkerMultiplier); worker++ {
@@ -379,6 +390,7 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
}()
}
+ // Notifier workers communicate detected issues to the user or any downstream systems.
// We want 1/4th of the notifier workers as the number of scanner workers.
const notifierWorkerRatio = 4
maxNotifierWorkers := 1
@@ -395,8 +407,6 @@ func Start(ctx context.Context, options ...EngineOption) (*Engine, error) {
e.notifyResults(ctx)
}()
}
-
- return e, nil
}
// Finish waits for running sources to complete and workers to finish scanning
@@ -445,64 +455,42 @@ type detectableChunk struct {
func (e *Engine) detectorWorker(ctx context.Context) {
var wgDetect sync.WaitGroup
+ // Reuse the same map to avoid allocations.
+ const avgDetectorsPerChunk = 2
+ chunkSpecificDetectors := make(map[detectorspb.DetectorType]detectorInfo, avgDetectorsPerChunk)
for originalChunk := range e.ChunksChan() {
for chunk := range sources.Chunker(originalChunk) {
- matchedKeywords := make(map[string]struct{})
atomic.AddUint64(&e.metrics.BytesScanned, uint64(len(chunk.Data)))
for _, decoder := range e.decoders {
- var decoderType detectorspb.DecoderType
- switch decoder.(type) {
- case *decoders.UTF8:
- decoderType = detectorspb.DecoderType_PLAIN
- case *decoders.Base64:
- decoderType = detectorspb.DecoderType_BASE64
- case *decoders.UTF16:
- decoderType = detectorspb.DecoderType_UTF16
- default:
- ctx.Logger().Info("unknown decoder type", "type", reflect.TypeOf(decoder).String())
- decoderType = detectorspb.DecoderType_UNKNOWN
- }
-
decoded := decoder.FromChunk(chunk)
-
if decoded == nil {
+ ctx.Logger().V(4).Info("no decoder found for chunk", "chunk", chunk)
continue
}
- // build a map of all keywords that were matched in the chunk
- for _, m := range e.prefilter.MatchString(strings.ToLower(string(decoded.Data))) {
- matchedKeywords[strings.ToLower(m.MatchString())] = struct{}{}
+ for _, match := range e.ahoCorasickCore.MatchString(string(decoded.Chunk.Data)) {
+ if !e.ahoCorasickCore.PopulateDetectorsByMatch(match, chunkSpecificDetectors) {
+ continue
+ }
}
- for verify, detectorsSet := range e.detectors {
- for _, detector := range detectorsSet {
- chunkContainsKeyword := false
- for _, kw := range detector.Keywords() {
- if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {
- chunkContainsKeyword = true
- break
- }
- }
-
- if !chunkContainsKeyword {
- continue
- }
-
- decoded.Verify = verify
- wgDetect.Add(1)
- e.detectableChunksChan <- detectableChunk{
- chunk: *decoded,
- detector: detector,
- decoder: decoderType,
- wgDoneFn: wgDetect.Done,
- }
+ for k, detector := range chunkSpecificDetectors {
+ decoded.Chunk.Verify = detector.shouldVerify
+ wgDetect.Add(1)
+ e.detectableChunksChan <- detectableChunk{
+ chunk: *decoded.Chunk,
+ detector: detector,
+ decoder: decoded.DecoderType,
+ wgDoneFn: wgDetect.Done,
}
+ delete(chunkSpecificDetectors, k)
}
}
}
atomic.AddUint64(&e.metrics.ChunksScanned, 1)
}
wgDetect.Wait()
+ ctx.Logger().V(4).Info("finished scanning chunks")
}
func (e *Engine) detectChunks(ctx context.Context) {
diff --git a/pkg/engine/git_test.go b/pkg/engine/git_test.go
index 821df94a2..ecfd1b589 100644
--- a/pkg/engine/git_test.go
+++ b/pkg/engine/git_test.go
@@ -2,6 +2,7 @@ package engine
import (
"os"
+ "runtime"
"testing"
"github.com/stretchr/testify/assert"
@@ -120,9 +121,10 @@ func BenchmarkGitEngine(b *testing.B) {
defer cancel()
e, err := Start(ctx,
- WithConcurrency(1),
+ WithConcurrency(uint8(runtime.NumCPU())),
WithDecoders(decoders.DefaultDecoders()...),
WithDetectors(false, DefaultDetectors()...),
+ WithPrinter(new(discardPrinter)),
)
assert.Nil(b, err)
@@ -133,6 +135,7 @@ func BenchmarkGitEngine(b *testing.B) {
}
}()
+ b.ResetTimer()
for i := 0; i < b.N; i++ {
// TODO: this is measuring the time it takes to initialize the source
// and not to do the full scan