Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[chore] Speedup IsKnownFalsePositive using sets (#2090)
Miccah600903f391fb
diff --git a/pkg/detectors/falsepositives.go b/pkg/detectors/falsepositives.goindex c12cf6b53..d77a65b40 100644--- a/pkg/detectors/falsepositives.go+++ b/pkg/detectors/falsepositives.go@@ -5,6 +5,7 @@ import ("math""strings""unicode"+ "unicode/utf8")var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}@@ -21,9 +22,9 @@ var wordList []bytevar programmingBookWords []bytetype Wordlists struct {- wordList []string- badList []string- programmingBookWords []string+ wordList map[string]struct{}+ badList map[string]struct{}+ programmingBookWords map[string]struct{}}var FalsePositiveWordlists = Wordlists{@@ -36,36 +37,29 @@ var FalsePositiveWordlists = Wordlists{// Currently that includes: No number, english word in key, or matches common example pattens.// Only the secret key material should be passed into this functionfunc IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {-+ if !utf8.ValidString(match) {+ return true+ }+ lower := strings.ToLower(match)for _, fp := range falsePositives {- if strings.Contains(strings.ToLower(match), string(fp)) {+ if strings.Contains(lower, string(fp)) {return true}}if wordCheck {// check against common substring badlist- if hasDictWord(FalsePositiveWordlists.badList, match) {+ if _, ok := FalsePositiveWordlists.badList[lower]; ok {return true}// check for dictionary word substrings- if hasDictWord(FalsePositiveWordlists.wordList, match) {+ if _, ok := FalsePositiveWordlists.wordList[lower]; ok {return true}// check for programming book token substrings- if hasDictWord(FalsePositiveWordlists.programmingBookWords, match) {- return true- }- }- return false-}--func hasDictWord(wordList []string, token string) bool {- lower := strings.ToLower(token)- for _, word := range wordList {- if strings.Contains(lower, word) {+ if _, ok := FalsePositiveWordlists.programmingBookWords[lower]; ok {return true}}@@ -82,11 +76,11 @@ func HasDigit(key string) bool {return false}-func bytesToCleanWordList(data []byte) []string {- words := []string{}+func bytesToCleanWordList(data []byte) map[string]struct{} {+ words := make(map[string]struct{})for _, word := range strings.Split(string(data), "\n") {if strings.TrimSpace(word) != "" {- words = append(words, strings.TrimSpace(strings.ToLower(word)))+ words[strings.TrimSpace(strings.ToLower(word))] = struct{}{}}}return wordsdiff --git a/pkg/detectors/falsepositives_test.go b/pkg/detectors/falsepositives_test.goindex b84aa427d..987269f45 100644--- a/pkg/detectors/falsepositives_test.go+++ b/pkg/detectors/falsepositives_test.go@@ -90,3 +90,10 @@ func TestStringShannonEntropy(t *testing.T) {})}}++func BenchmarkDefaultIsKnownFalsePositive(b *testing.B) {+ for i := 0; i < b.N; i++ {+ // Use a string that won't be found in any dictionary for the worst case check.+ IsKnownFalsePositive("aoeuaoeuaoeuaoeuaoeuaoeu", DefaultFalsePositives, true)+ }+}