Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Improve fp ignore logic (#2351)
Dustin Decker7befefd369f6
diff --git a/go.mod b/go.modindex 915abcfd3..57b42b313 100644--- a/go.mod+++ b/go.mod@@ -39,6 +39,7 @@ require (github.com/go-sql-driver/mysql v1.7.1github.com/gobwas/glob v0.2.3github.com/golang-jwt/jwt v3.2.2+incompatible+ github.com/golang-jwt/jwt/v4 v4.5.0github.com/google/go-cmp v0.6.0github.com/google/go-containerregistry v0.17.0github.com/google/go-github/v42 v42.0.0@@ -166,7 +167,6 @@ require (github.com/go-ole/go-ole v1.2.6 // indirectgithub.com/goccy/go-json v0.10.0 // indirectgithub.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect- github.com/golang-jwt/jwt/v4 v4.5.0 // indirectgithub.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirectgithub.com/golang-sql/sqlexp v0.1.0 // indirectgithub.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirectdiff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.goindex 27e21951f..dbb7a3962 100644--- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go+++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go@@ -52,7 +52,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1 := detectors.Result{DetectorType: detectorspb.DetectorType_AzureSearchQueryKey,- Raw: []byte(resMatch + resUrlMatch),+ Raw: []byte(resMatch),+ RawV2: []byte(resMatch + resUrlMatch),}if verify {client := s.clientdiff --git a/pkg/detectors/badlist.txt b/pkg/detectors/badlist.txtindex 274a32cea..713871d7e 100644--- a/pkg/detectors/badlist.txt+++ b/pkg/detectors/badlist.txt@@ -4,12 +4,7 @@ arrayuintbooleanconfig-/>-</-\n-\rparse-()funcparamcancel@@ -27,7 +22,6 @@ spaceidentblocktype-\"indexcasesafe@@ -87,8 +81,6 @@ keywordtracetruncategroup-a-z-0-9hrefscalemodel@@ -106,26 +98,18 @@ closedeferstart;var-":storageblobcred-${math.xmlconflict-];-$(--{{hack--v1--v2packagecontractschemavec<ed25519-(&prefixsuffixcompress@@ -177,7 +161,6 @@ errorrevokeencryptbinary-md52018-2019-2020-@@ -188,14 +171,12 @@ rootreadontest2048-1<<matchprivatekey_aes256aes128state-...allocprototerm@@ -281,8 +262,6 @@ k8s.roleapplicationexplic-[[-]]randomDES33DES@@ -295,7 +274,6 @@ tag:extendsplitoption-t=0fontsize"keyboard@@ -306,5 +284,4 @@ iphonedevelopmasterslave-secondary----+secondary\ No newline at end of filediff --git a/pkg/detectors/falsepositives.go b/pkg/detectors/falsepositives.goindex 948ad12d9..a848267da 100644--- a/pkg/detectors/falsepositives.go+++ b/pkg/detectors/falsepositives.go@@ -6,6 +6,8 @@ import ("strings""unicode""unicode/utf8"++ ahocorasick "github.com/BobuSumisu/aho-corasick")var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcde", "00000", "sample", "www"}@@ -21,16 +23,21 @@ var wordList []byte//go:embed "programmingbooks.txt"var programmingBookWords []byte-type Wordlists struct {- wordList map[string]struct{}- badList map[string]struct{}- programmingBookWords map[string]struct{}-}+var filter *ahocorasick.Trie++func init() {+ builder := ahocorasick.NewTrieBuilder()++ wordList := bytesToCleanWordList(wordList)+ builder.AddStrings(wordList)-var FalsePositiveWordlists = Wordlists{- wordList: bytesToCleanWordList(wordList),- badList: bytesToCleanWordList(badList),- programmingBookWords: bytesToCleanWordList(programmingBookWords),+ badList := bytesToCleanWordList(badList)+ builder.AddStrings(badList)++ programmingBookWords := bytesToCleanWordList(programmingBookWords)+ builder.AddStrings(programmingBookWords)++ filter = builder.Build()}// IsKnownFalsePositives will not return a valid secret finding if any of the disqualifying conditions are met@@ -48,21 +55,11 @@ func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordChec}if wordCheck {- // check against common substring badlist- if _, ok := FalsePositiveWordlists.badList[lower]; ok {- return true- }-- // check for dictionary word substrings- if _, ok := FalsePositiveWordlists.wordList[lower]; ok {- return true- }-- // check for programming book token substrings- if _, ok := FalsePositiveWordlists.programmingBookWords[lower]; ok {+ if filter.MatchFirstString(lower) != nil {return true}}+return false}@@ -76,14 +73,19 @@ func HasDigit(key string) bool {return false}-func bytesToCleanWordList(data []byte) map[string]struct{} {+func bytesToCleanWordList(data []byte) []string {words := make(map[string]struct{})for _, word := range strings.Split(string(data), "\n") {if strings.TrimSpace(word) != "" {words[strings.TrimSpace(strings.ToLower(word))] = struct{}{}}}- return words++ wordList := make([]string, 0, len(words))+ for word := range words {+ wordList = append(wordList, word)+ }+ return wordList}func StringShannonEntropy(input string) float64 {diff --git a/pkg/detectors/falsepositives_test.go b/pkg/detectors/falsepositives_test.goindex 987269f45..216ce87b9 100644--- a/pkg/detectors/falsepositives_test.go+++ b/pkg/detectors/falsepositives_test.go@@ -12,6 +12,7 @@ func TestIsFalsePositive(t *testing.T) {type args struct {match stringfalsePositives []FalsePositive+ useWordlist bool}tests := []struct {name string@@ -23,21 +24,50 @@ func TestIsFalsePositive(t *testing.T) {args: args{match: "example",falsePositives: DefaultFalsePositives,+ useWordlist: false,},want: true,},+ {+ name: "fp - in wordlist",+ args: args{+ match: "sdfdsfprivatesfsdfd",+ falsePositives: DefaultFalsePositives,+ useWordlist: true,+ },+ want: true,+ },+ {+ name: "fp - not in wordlist",+ args: args{+ match: "sdfdsfsfsdfd",+ falsePositives: DefaultFalsePositives,+ useWordlist: true,+ },+ want: false,+ },{name: "not fp",args: args{match: "notafp123",falsePositives: DefaultFalsePositives,+ useWordlist: false,},want: false,},+ {+ name: "fp - in wordlist exact match",+ args: args{+ match: "private",+ falsePositives: DefaultFalsePositives,+ useWordlist: true,+ },+ want: true,+ },}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- if got := IsKnownFalsePositive(tt.args.match, tt.args.falsePositives, false); got != tt.want {+ if got := IsKnownFalsePositive(tt.args.match, tt.args.falsePositives, tt.args.useWordlist); got != tt.want {t.Errorf("IsKnownFalsePositive() = %v, want %v", got, tt.want)}})diff --git a/pkg/detectors/programmingbooks.txt b/pkg/detectors/programmingbooks.txtindex ed57aa867..43681cc82 100644--- a/pkg/detectors/programmingbooks.txt+++ b/pkg/detectors/programmingbooks.txt@@ -15,8 +15,7 @@ ${ts.map(kv${uname$value$x:expr-+3=err-a;++3=erra][appendix_aabbreviatedabcabcabc