Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Keyword optimization (#1144)
Zachary Rice4777b77ec6f9
diff --git a/.gitignore b/.gitignoreindex 416391864..e9a3dd8a8 100644--- a/.gitignore+++ b/.gitignore@@ -1,4 +1,7 @@.ideadist.env-*.test\ No newline at end of file+*.test++# binary+trufflehogdiff --git a/go.mod b/go.modindex 4a810d439..03a726868 100644--- a/go.mod+++ b/go.mod@@ -120,6 +120,7 @@ require (github.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirectgithub.com/onsi/ginkgo v1.16.5 // indirectgithub.com/onsi/gomega v1.23.0 // indirect+ github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirectgithub.com/pierrec/lz4/v4 v4.1.14 // indirectgithub.com/pjbgf/sha1cd v0.2.3 // indirectgithub.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirectdiff --git a/go.sum b/go.sumindex 7b61144b1..2304f479f 100644--- a/go.sum+++ b/go.sum@@ -281,6 +281,8 @@ github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8=+github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw=github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 6b179adec..211e79c1e 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -9,6 +9,7 @@ import ("sync/atomic""time"+ ahocorasick "github.com/petar-dambovaliev/aho-corasick""google.golang.org/protobuf/proto""github.com/trufflesecurity/trufflehog/v3/pkg/common"@@ -36,6 +37,10 @@ type Engine struct {// If there are multiple unverified results for the same chunk for the same detector,// only the first one will be kept.filterUnverified bool++ // prefilter is a ahocorasick struct used for doing efficient string+ // matching given a set of words (keywords from the rules in the config)+ prefilter ahocorasick.AhoCorasick}type EngineOption func(*Engine)@@ -110,7 +115,6 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {}// Set defaults.-if e.concurrency == 0 {numCPU := runtime.NumCPU()ctx.Logger().Info("No concurrency specified, defaulting to max", "cpu", numCPU)@@ -128,6 +132,23 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {e.detectors[false] = []detectors.Detector{}}+ // build ahocorasick prefilter for efficient string matching+ // on keywords+ keywords := []string{}+ for _, d := range e.detectors[false] {+ keywords = append(keywords, d.Keywords()...)+ }+ for _, d := range e.detectors[true] {+ keywords = append(keywords, d.Keywords()...)+ }+ builder := ahocorasick.NewAhoCorasickBuilder(ahocorasick.Opts{+ AsciiCaseInsensitive: true,+ MatchOnlyWholeWords: false,+ MatchKind: ahocorasick.LeftMostLongestMatch,+ DFA: true,+ })+ e.prefilter = builder.Build(keywords)+ctx.Logger().V(2).Info("loaded decoders", "count", len(e.decoders))ctx.Logger().V(2).Info("loaded detectors","total", len(e.detectors[true])+len(e.detectors[false]),@@ -208,6 +229,7 @@ func (e *Engine) DetectorAvgTime() map[string][]time.Duration {func (e *Engine) detectorWorker(ctx context.Context) {for originalChunk := range e.chunks {for chunk := range sources.Chunker(originalChunk) {+ matchedKeywords := make(map[string]struct{})atomic.AddUint64(&e.bytesScanned, uint64(len(chunk.Data)))for _, decoder := range e.decoders {var decoderType detectorspb.DecoderType@@ -224,21 +246,29 @@ func (e *Engine) detectorWorker(ctx context.Context) {if decoded == nil {continue}+dataLower := strings.ToLower(string(decoded.Data))+ matches := e.prefilter.FindAll(dataLower)++ for _, m := range matches {+ matchedKeywords[dataLower[m.Start():m.End()]] = struct{}{}+ }+for verify, detectorsSet := range e.detectors {for _, detector := range detectorsSet {- start := time.Now()- foundKeyword := false+ chunkContainsKeyword := falsefor _, kw := range detector.Keywords() {- if strings.Contains(dataLower, strings.ToLower(kw)) {- foundKeyword = true- break+ if _, ok := matchedKeywords[strings.ToLower(kw)]; ok {+ chunkContainsKeyword = true}}- if !foundKeyword {++ if !chunkContainsKeyword {continue}+ start := time.Now()+results, err := func() ([]detectors.Result, error) {ctx, cancel := context.WithTimeout(ctx, time.Second*10)defer cancel()