Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
feat(detectors): log falsepositive reason (#2969)
Richard Gomez2964b3b2d2ed
diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.goindex c914caa26..32b6af8c4 100644--- a/pkg/custom_detectors/custom_detectors.go+++ b/pkg/custom_detectors/custom_detectors.go@@ -8,11 +8,12 @@ import ("regexp""strings"+ "golang.org/x/sync/errgroup"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"- "golang.org/x/sync/errgroup")// The maximum number of matches from one chunk. This const is used when@@ -110,8 +111,8 @@ func (c *CustomRegexWebhook) FromData(ctx context.Context, verify bool, data []breturn results, nil}-func (c *CustomRegexWebhook) IsFalsePositive(_ detectors.Result) bool {- return false+func (c *CustomRegexWebhook) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}func (c *CustomRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {diff --git a/pkg/detectors/azurebatch/azurebatch.go b/pkg/detectors/azurebatch/azurebatch.goindex 342b37d13..da5fd9333 100644--- a/pkg/detectors/azurebatch/azurebatch.go+++ b/pkg/detectors/azurebatch/azurebatch.go@@ -107,8 +107,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool {- return false+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go b/pkg/detectors/azurecontainerregistry/azurecontainerregistry.goindex 0126d4745..80fd9ce48 100644--- a/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go+++ b/pkg/detectors/azurecontainerregistry/azurecontainerregistry.go@@ -96,8 +96,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool {- return false+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/detectors/falsepositives.go b/pkg/detectors/falsepositives.goindex b3ffde587..896312c1c 100644--- a/pkg/detectors/falsepositives.go+++ b/pkg/detectors/falsepositives.go@@ -17,7 +17,10 @@ var DefaultFalsePositives = []FalsePositive{"example", "xxxxxx", "aaaaaa", "abcdtype FalsePositive stringtype CustomFalsePositiveChecker interface {- IsFalsePositive(result Result) bool+ // IsFalsePositive returns two values:+ // 1. Whether the result is a false positive.+ // 2. If #1 is `true`, the reason why.+ IsFalsePositive(result Result) (bool, string)}//go:embed "badlist.txt"@@ -46,38 +49,40 @@ func init() {filter = builder.Build()}-func GetFalsePositiveCheck(detector Detector) func(Result) bool {+func GetFalsePositiveCheck(detector Detector) func(Result) (bool, string) {checker, ok := detector.(CustomFalsePositiveChecker)if ok {return checker.IsFalsePositive}- return func(res Result) bool {+ return func(res Result) (bool, string) {return IsKnownFalsePositive(string(res.Raw), DefaultFalsePositives, true)}}-// IsKnownFalsePositive will not return a valid secret finding if any of the disqualifying conditions are met-// Currently that includes: No number, english word in key, or matches common example patterns.+// IsKnownFalsePositive returns whether a finding is (likely) a known false positive, and the reason for the detection.+//+// Currently, this includes: english word in key or matches common example patterns.// Only the secret key material should be passed into this function-func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) bool {+func IsKnownFalsePositive(match string, falsePositives []FalsePositive, wordCheck bool) (bool, string) {if !utf8.ValidString(match) {- return true+ return true, "invalid utf8"}lower := strings.ToLower(match)for _, fp := range falsePositives {- if strings.Contains(lower, string(fp)) {- return true+ fps := string(fp)+ if strings.Contains(lower, fps) {+ return true, "matches term: " + fps}}if wordCheck {- if filter.MatchFirstString(lower) != nil {- return true+ if m := filter.MatchFirstString(lower); m != nil {+ return true, "matches wordlist: " + m.MatchString()}}- return false+ return false, ""}func HasDigit(key string) bool {@@ -153,10 +158,11 @@ func FilterKnownFalsePositives(ctx context.Context, detector Detector, results [for _, result := range results {if !result.Verified && result.Raw != nil {- if !isFalsePositive(result) {+ isFp, reason := isFalsePositive(result)+ if !isFp {filteredResults = append(filteredResults, result)} else if shouldLog {- ctx.Logger().Info("Filtered out known false positive", "result", result)+ ctx.Logger().Info("Filtered out known false positive", "result", result, "reason", reason)}} else {filteredResults = append(filteredResults, result)diff --git a/pkg/detectors/falsepositives_test.go b/pkg/detectors/falsepositives_test.goindex e4222373f..77bb99a1a 100644--- a/pkg/detectors/falsepositives_test.go+++ b/pkg/detectors/falsepositives_test.go@@ -9,6 +9,7 @@ import ("testing""github.com/stretchr/testify/assert"+logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -28,7 +29,7 @@ func (d fakeDetector) Type() detectorspb.DetectorType {return detectorspb.DetectorType(0)}-func (d customFalsePositiveChecker) IsFalsePositive(result Result) bool {+func (d customFalsePositiveChecker) IsFalsePositive(result Result) (bool, string) {return IsKnownFalsePositive(string(result.Raw), []FalsePositive{"a specific magic string"}, false)}@@ -120,7 +121,7 @@ func TestIsFalsePositive(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- if got := IsKnownFalsePositive(tt.args.match, tt.args.falsePositives, tt.args.useWordlist); got != tt.want {+ if got, _ := IsKnownFalsePositive(tt.args.match, tt.args.falsePositives, tt.args.useWordlist); got != tt.want {t.Errorf("IsKnownFalsePositive() = %v, want %v", got, tt.want)}})diff --git a/pkg/detectors/ftp/ftp.go b/pkg/detectors/ftp/ftp.goindex 8a689f52b..af3ea40fa 100644--- a/pkg/detectors/ftp/ftp.go+++ b/pkg/detectors/ftp/ftp.go@@ -103,7 +103,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) IsFalsePositive(result detectors.Result) bool {+func (s Scanner) IsFalsePositive(result detectors.Result) (bool, string) {return detectors.IsKnownFalsePositive(string(result.Raw), []detectors.FalsePositive{"@ftp.freebsd.org"}, false)}diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.goindex e031553e4..21c204d79 100644--- a/pkg/detectors/gcp/gcp.go+++ b/pkg/detectors/gcp/gcp.go@@ -133,8 +133,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool {- return false+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.goindex 334031187..b5eb37e9a 100644--- a/pkg/detectors/github/v1/github_old.go+++ b/pkg/detectors/github/v1/github_old.go@@ -74,7 +74,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result// Note that this false positive check happens **before** verification! I don't know why it's written this way// but that's why this logic wasn't moved into a CustomFalsePositiveChecker implementation.specificFPs := []detectors.FalsePositive{"github commit"}- if detectors.IsKnownFalsePositive(token, specificFPs, false) {+ if isFp, _ := detectors.IsKnownFalsePositive(token, specificFPs, false); isFp {continue}diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.goindex ef097d5b5..d87e64377 100644--- a/pkg/detectors/jdbc/jdbc.go+++ b/pkg/detectors/jdbc/jdbc.go@@ -106,8 +106,8 @@ matchLoop:return}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool {- return false+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}func tryRedactAnonymousJDBC(conn string) string {diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.goindex bb48b1a40..8593fae01 100644--- a/pkg/detectors/mongodb/mongodb.go+++ b/pkg/detectors/mongodb/mongodb.go@@ -73,8 +73,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool {- return false+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}func isErrDeterminate(err error) bool {diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.goindex 7bb402dcd..b1600ab4f 100644--- a/pkg/detectors/postgres/postgres.go+++ b/pkg/detectors/postgres/postgres.go@@ -148,8 +148,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detereturn results, nil}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool {- return false+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}func findUriMatches(data []byte) []map[string]string {diff --git a/pkg/detectors/privatekey/privatekey.go b/pkg/detectors/privatekey/privatekey.goindex 715a82a76..b9519c369 100644--- a/pkg/detectors/privatekey/privatekey.go+++ b/pkg/detectors/privatekey/privatekey.go@@ -151,7 +151,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool { return false }+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""+}type result struct {CertificateURLs []stringdiff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.goindex 7ad1592d1..00ac455bb 100644--- a/pkg/detectors/shopify/shopify.go+++ b/pkg/detectors/shopify/shopify.go@@ -13,7 +13,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}@@ -91,8 +91,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool {- return false+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}type shopifyTokenAccessScopes struct {diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.goindex 11bdcebd7..cd8af3004 100644--- a/pkg/detectors/uri/uri.go+++ b/pkg/detectors/uri/uri.go@@ -103,8 +103,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) IsFalsePositive(_ detectors.Result) bool {- return false+func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {+ return false, ""}func verifyURL(ctx context.Context, client *http.Client, u *url.URL) (bool, error) {diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 07fe69331..a1f020479 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -1056,7 +1056,7 @@ func (e *Engine) processResult(ctx context.Context,data detectableChunk,res detectors.Result,- isFalsePositive func(detectors.Result) bool,+ isFalsePositive func(detectors.Result) (bool, string),) {ignoreLinePresent := falseif SupportsLineNumbers(data.chunk.SourceType) {@@ -1081,7 +1081,8 @@ func (e *Engine) processResult(secret.DecoderType = data.decoderif !res.Verified && res.Raw != nil {- secret.IsWordlistFalsePositive = isFalsePositive(res)+ isFp, _ := isFalsePositive(res)+ secret.IsWordlistFalsePositive = isFp}e.results <- secret