Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Expose detector-specific false positive logic (#2743)
Cody Rose2f7029bc4d37
diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.goindex 4e6b8feea..c914caa26 100644--- a/pkg/custom_detectors/custom_detectors.go+++ b/pkg/custom_detectors/custom_detectors.go@@ -29,6 +29,7 @@ type CustomRegexWebhook struct {// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*CustomRegexWebhook)(nil)+var _ detectors.CustomFalsePositiveChecker = (*CustomRegexWebhook)(nil)// NewWebhookCustomRegex initializes and validates a CustomRegexWebhook. An// unexported type is intentionally returned here to ensure the values have@@ -109,6 +110,10 @@ func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []breturn results, nil}+func (c *CustomRegexWebhook) IsFalsePositive(_ detectors.Result) bool {+ return false+}+func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {if common.IsDone(ctx) {// TODO: Log we're possibly leaving out results.diff --git a/pkg/detectors/azurebatch/azurebatch.go b/pkg/detectors/azurebatch/azurebatch.goindex f04b907e3..bd9e1936f 100644--- a/pkg/detectors/azurebatch/azurebatch.go+++ b/pkg/detectors/azurebatch/azurebatch.go@@ -23,6 +23,7 @@ type Scanner struct {// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (defaultClient = common.SaneHttpClient()@@ -105,6 +106,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {+ return false+}+func (s Scanner) Type() detectorspb.DetectorType {return detectorspb.DetectorType_AzureBatch}diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry.goindex b0651ab92..55c0f4b3f 100644--- a/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go+++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go@@ -19,6 +19,7 @@ type Scanner struct {// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (defaultClient = common.SaneHttpClient()@@ -94,6 +95,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {+ return false+}+func (s Scanner) Type() detectorspb.DetectorType {return detectorspb.DetectorType_AzureContainerRegistry}diff --git a/pkg/detectors/falsepositives.go b/pkg/detectors/falsepositives.goindex 29ce2e7c8..1b3594a6d 100644--- a/pkg/detectors/falsepositives.go+++ b/pkg/detectors/falsepositives.go@@ -8,8 +8,6 @@ import ("unicode/utf8"ahocorasick "github.com/BobuSumisu/aho-corasick"- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"-"github.com/trufflesecurity/trufflehog/v3/pkg/context")@@ -17,6 +15,10 @@ var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcdtype FalsePositive string+type CustomFalsePositiveChecker interface {+ IsFalsePositive(result Result) bool+}+//go:embed "badlist.txt"var badList []byte@@ -43,6 +45,17 @@ func init() {filter = builder.Build()}+func GetFalsePositiveCheck(detector Detector) func(Result) bool {+ checker, ok := detector.(CustomFalsePositiveChecker)+ if ok {+ return checker.IsFalsePositive+ }++ return func(res Result) bool {+ return IsKnownFalsePositive(string(res.Raw), DefaultFalsePositives, true)+ }+}+// IsKnownFalsePositive will not return a valid secret finding if any of the disqualifying conditions are met// Currently that includes: No number, english word in key, or matches common example pattens.// Only the secret key material should be passed into this function@@ -132,34 +145,17 @@ func FilterResultsWithEntropy(ctx context.Context, results []Result, entropy flo}// FilterKnownFalsePositives filters out known false positives from the results.-func FilterKnownFalsePositives(ctx context.Context, results []Result, falsePositives []FalsePositive, wordCheck bool, shouldLog bool) []Result {+func FilterKnownFalsePositives(ctx context.Context, detector Detector, results []Result, shouldLog bool) []Result {var filteredResults []Result++ isFalsePositive := GetFalsePositiveCheck(detector)+for _, result := range results {- if !result.Verified {- switch result.DetectorType {- case detectorspb.DetectorType_CustomRegex:- filteredResults = append(filteredResults, result)- case detectorspb.DetectorType_GCP,- detectorspb.DetectorType_URI,- detectorspb.DetectorType_AzureBatch,- detectorspb.DetectorType_AzureContainerRegistry,- detectorspb.DetectorType_Shopify,- detectorspb.DetectorType_Postgres,- detectorspb.DetectorType_MongoDB,- detectorspb.DetectorType_JDBC:+ if !result.Verified && result.Raw != nil {+ if !isFalsePositive(result) {filteredResults = append(filteredResults, result)- default:- if result.Raw != nil {- if !IsKnownFalsePositive(string(result.Raw), falsePositives, wordCheck) {- filteredResults = append(filteredResults, result)- } else {- if shouldLog {- ctx.Logger().Info("Filtered out known false positive", "result", result)- }- }- } else {- filteredResults = append(filteredResults, result)- }+ } else if shouldLog {+ ctx.Logger().Info("Filtered out known false positive", "result", result)}} else {filteredResults = append(filteredResults, result)diff --git a/pkg/detectors/falsepositives_test.go b/pkg/detectors/falsepositives_test.goindex 216ce87b9..e4222373f 100644--- a/pkg/detectors/falsepositives_test.go+++ b/pkg/detectors/falsepositives_test.go@@ -4,10 +4,63 @@package detectorsimport (+ "context"_ "embed""testing"++ "github.com/stretchr/testify/assert"+ logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")+type fakeDetector struct{}+type customFalsePositiveChecker struct{ fakeDetector }++func (d fakeDetector) FromData(ctx context.Context, verify bool, data []byte) ([]Result, error) {+ return nil, nil+}++func (d fakeDetector) Keywords() []string {+ return nil+}++func (d fakeDetector) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType(0)+}++func (d customFalsePositiveChecker) IsFalsePositive(result Result) bool {+ return IsKnownFalsePositive(string(result.Raw), []FalsePositive{"a specific magic string"}, false)+}++func TestFilterKnownFalsePositives_DefaultLogic(t *testing.T) {+ results := []Result{+ {Raw: []byte("00000")}, // "default" false positive list+ {Raw: []byte("number")}, // from wordlist+ {Raw: []byte("hga8adshla3434g")}, // real secret+ }+ expected := []Result{+ {Raw: []byte("hga8adshla3434g")},+ }+ filtered := FilterKnownFalsePositives(logContext.Background(), fakeDetector{}, results, false)+ assert.ElementsMatch(t, expected, filtered)+}++func TestFilterKnownFalsePositives_CustomLogic(t *testing.T) {+ results := []Result{+ {Raw: []byte("a specific magic string")}, // specific target+ {Raw: []byte("00000")}, // "default" false positive list+ {Raw: []byte("number")}, // from wordlist+ {Raw: []byte("hga8adshla3434g")}, // real secret+ }+ expected := []Result{+ {Raw: []byte("00000")},+ {Raw: []byte("number")},+ {Raw: []byte("hga8adshla3434g")},+ }+ filtered := FilterKnownFalsePositives(logContext.Background(), customFalsePositiveChecker{}, results, false)+ assert.ElementsMatch(t, expected, filtered)+}+func TestIsFalsePositive(t *testing.T) {type args struct {match stringdiff --git a/pkg/detectors/ftp/ftp.go b/pkg/detectors/ftp/ftp.goindex 4a76d2441..8a689f52b 100644--- a/pkg/detectors/ftp/ftp.go+++ b/pkg/detectors/ftp/ftp.go@@ -30,6 +30,7 @@ type Scanner struct {// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (keyPat = regexp.MustCompile(`\bftp://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)@@ -96,16 +97,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}}- if detectors.IsKnownFalsePositive(string(s1.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false) {- continue- }-results = append(results, s1)}return results, nil}+func (s Scanner) IsFalsePositive(result detectors.Result) bool {+ return detectors.IsKnownFalsePositive(string(result.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false)+}+func isErrDeterminate(e error) bool {ftpErr := &textproto.Error{}return errors.As(e, &ftpErr) && ftpErr.Code == ftpNotLoggedIndiff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.goindex 6d1993e62..6a2b3e1da 100644--- a/pkg/detectors/gcp/gcp.go+++ b/pkg/detectors/gcp/gcp.go@@ -18,6 +18,7 @@ type Scanner struct{}// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (keyPat = regexp.MustCompile(`\{[^{]+auth_provider_x509_cert_url[^}]+\}`)@@ -120,6 +121,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn}+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {+ return false+}+func (s Scanner) Type() detectorspb.DetectorType {return detectorspb.DetectorType_GCP}diff --git a/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go b/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.goindex eeecab957..debe41fa6 100644--- a/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go+++ b/pkg/detectors/gcpapplicationdefaultcredentials/gcpapplicationdefaultcredentials.go@@ -82,11 +82,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1.SetVerificationError(verificationErr, match)}- // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.- if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {- continue- }-results = append(results, s1)}diff --git a/pkg/detectors/generic/generic.go b/pkg/detectors/generic/generic.goindex 4037a15aa..327a139d1 100644--- a/pkg/detectors/generic/generic.go+++ b/pkg/detectors/generic/generic.go@@ -75,10 +75,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result// Least expensive-> most expensive filters.// Substrings, then patterns.- if detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {- continue- }-// toss any that match regexesif hasReMatch(s.excludeMatchers, token) {continuediff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.goindex 6f7b56d26..3ade673cf 100644--- a/pkg/detectors/github/v1/github_old.go+++ b/pkg/detectors/github/v1/github_old.go@@ -69,6 +69,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resulttoken := match[1]+ // Note that this false positive check happens **before** verification! I don't know why it's written this way+ // but that's why this logic wasn't moved into a CustomFalsePositiveChecker implementation.specificFPs := []detectors.FalsePositive{"github commit"}if detectors.IsKnownFalsePositive(token, specificFPs, false) {continuediff --git a/pkg/detectors/github/v2/github.go b/pkg/detectors/github/v2/github.goindex 912c17a20..2c21a3ce4 100644--- a/pkg/detectors/github/v2/github.go+++ b/pkg/detectors/github/v2/github.go@@ -83,9 +83,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}}- if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {- continue- }results = append(results, s1)}diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.goindex cc28db559..4b6cee99f 100644--- a/pkg/detectors/jdbc/jdbc.go+++ b/pkg/detectors/jdbc/jdbc.go@@ -45,6 +45,7 @@ func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (keyPat = regexp.MustCompile(`(?i)jdbc:[\w]{3,10}:[^\s"']{0,512}`)@@ -98,14 +99,16 @@ matchLoop:// TODO: specialized redaction}--results = append(results, s)}return}+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {+ return false+}+func tryRedactAnonymousJDBC(conn string) string {if s, ok := tryRedactURLParams(conn); ok {return sdiff --git a/pkg/detectors/jupiterone/jupiterone.go b/pkg/detectors/jupiterone/jupiterone.goindex 2e1d0eaf0..2bf322bb1 100644--- a/pkg/detectors/jupiterone/jupiterone.go+++ b/pkg/detectors/jupiterone/jupiterone.go@@ -83,11 +83,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}}- // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.- if !s1.Verified && detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {- continue- }-results = append(results, s1)}diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.goindex 6e9a8ea65..bb48b1a40 100644--- a/pkg/detectors/mongodb/mongodb.go+++ b/pkg/detectors/mongodb/mongodb.go@@ -24,6 +24,7 @@ type Scanner struct {// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (defaultTimeout = 2 * time.Second@@ -72,6 +73,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {+ return false+}+func isErrDeterminate(err error) bool {switch e := err.(type) {case topology.ConnectionError:diff --git a/pkg/detectors/onfleet/onfleet.go b/pkg/detectors/onfleet/onfleet.goindex cb1060b0d..5e348345f 100644--- a/pkg/detectors/onfleet/onfleet.go+++ b/pkg/detectors/onfleet/onfleet.go@@ -60,11 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1.SetVerificationError(verificationErr, match)}- // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.- if !s1.Verified && detectors.IsKnownFalsePositive(match, detectors.DefaultFalsePositives, true) {- continue- }-results = append(results, s1)}diff --git a/pkg/detectors/pagarme/pagarme.go b/pkg/detectors/pagarme/pagarme.goindex 5cc5be351..af1fca76e 100644--- a/pkg/detectors/pagarme/pagarme.go+++ b/pkg/detectors/pagarme/pagarme.go@@ -60,11 +60,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1.SetVerificationError(verificationErr, match)}- // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.- if !s1.Verified && detectors.IsKnownFalsePositive(match, detectors.DefaultFalsePositives, true) {- continue- }-results = append(results, s1)}diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.goindex 09ec98d92..4f1a5b8e3 100644--- a/pkg/detectors/postgres/postgres.go+++ b/pkg/detectors/postgres/postgres.go@@ -55,6 +55,9 @@ type Scanner struct {detectLoopback bool // Automated tests run against localhost, but we want to ignore those results in the wild}+var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)+func (s Scanner) Keywords() []string {return []string{"postgres"}}@@ -144,6 +147,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detereturn results, nil}+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {+ return false+}+func findUriMatches(data []byte) []map[string]string {var matches []map[string]stringfor _, uri := range uriPattern.FindAll(data, -1) {diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.goindex 187889f17..115d3d4df 100644--- a/pkg/detectors/shopify/shopify.go+++ b/pkg/detectors/shopify/shopify.go@@ -3,10 +3,11 @@ package shopifyimport ("context""encoding/json"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -16,6 +17,7 @@ type Scanner struct{}// Ensure the Scanner satisfies the interface at compile timevar _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (client = common.SaneHttpClient()@@ -87,6 +89,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {+ return false+}+type shopifyTokenAccessScopes struct {AccessScopes []struct {Handle string `json:"handle"`diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.goindex 33d63ad77..8015b0f21 100644--- a/pkg/detectors/uri/uri.go+++ b/pkg/detectors/uri/uri.go@@ -2,12 +2,13 @@ package uriimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""net/url""strings""time"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -20,6 +21,7 @@ type Scanner struct {// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)@@ -101,6 +103,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}+func (s Scanner) IsFalsePositive(_ detectors.Result) bool {+ return false+}+func verifyURL(ctx context.Context, client *http.Client, u *url.URL) (bool, error) {// defuse most SSRF payloadsu.Path = strings.TrimSuffix(u.Path, "/")diff --git a/pkg/detectors/wiz/wiz.go b/pkg/detectors/wiz/wiz.goindex e5bde3842..c0de255cf 100644--- a/pkg/detectors/wiz/wiz.go+++ b/pkg/detectors/wiz/wiz.go@@ -69,14 +69,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1.SetVerificationError(verificationErr, idMatch, secretMatch)}- // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key.- if !s1.Verified {- if detectors.IsKnownFalsePositive(idMatch, detectors.DefaultFalsePositives, true) ||- detectors.IsKnownFalsePositive(secretMatch, detectors.DefaultFalsePositives, true) {- continue- }- }-results = append(results, s1)// If we've found a verified match with this ID, we don't need to look for anymore. So move on to the next ID.diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 22cc18686..1798a2467 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -831,7 +831,7 @@ func (e *Engine) detectChunk(ctx context.Context, data detectableChunk) {results = detectors.CleanResults(results)}- results = detectors.FilterKnownFalsePositives(ctx, results, detectors.DefaultFalsePositives, true, e.logFilteredUnverified)+ results = detectors.FilterKnownFalsePositives(ctx, data.detector, results, e.logFilteredUnverified)if e.filterEntropy != nil {results = detectors.FilterResultsWithEntropy(ctx, results, *e.filterEntropy, e.logFilteredUnverified)