Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Customize results cleaning (using smuggled interface) (#3235)
Cody Rosef39a5254ffd4
diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.goindex 2527e9236..0416f051e 100644--- a/pkg/detectors/aws/aws.go+++ b/pkg/detectors/aws/aws.go@@ -105,6 +105,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {// Ensure the scanner satisfies the interface at compile time.var _ detectors.Detector = (*scanner)(nil)var _ detectors.MultiPartCredentialProvider = (*scanner)(nil)+var _ detectors.CustomResultsCleaner = (*scanner)(nil)var (defaultVerificationClient = common.SaneHttpClient()@@ -245,7 +246,11 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result}}}- return awsCustomCleanResults(results), nil+ return results, nil+}++func (s scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool {+ return true}func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {@@ -399,7 +404,7 @@ func (s scanner) verifyCanary(resIDMatch, resSecretMatch string) (bool, string,}}-func awsCustomCleanResults(results []detectors.Result) []detectors.Result {+func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {if len(results) == 0 {return results}diff --git a/pkg/detectors/awssessionkeys/awssessionkey.go b/pkg/detectors/awssessionkeys/awssessionkey.goindex 4974f3d76..4cd0afdba 100644--- a/pkg/detectors/awssessionkeys/awssessionkey.go+++ b/pkg/detectors/awssessionkeys/awssessionkey.go@@ -49,6 +49,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {// Ensure the scanner satisfies the interface at compile time.var _ detectors.Detector = (*scanner)(nil)+var _ detectors.CustomResultsCleaner = (*scanner)(nil)var (defaultVerificationClient = common.SaneHttpClient()@@ -167,7 +168,11 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result}}}- return awsCustomCleanResults(results), nil+ return results, nil+}++func (s scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool {+ return true}func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, resSessionMatch string, retryOn403 bool) (bool, map[string]string, error) {@@ -283,7 +288,7 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str}}-func awsCustomCleanResults(results []detectors.Result) []detectors.Result {+func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {if len(results) == 0 {return results}diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex b3e975ff8..b4f643f97 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -26,6 +26,22 @@ type Detector interface {Type() detectorspb.DetectorType}+// CustomResultsCleaner is an optional interface that a detector can implement to customize how its generated results+// are "cleaned," which is defined as removing superfluous results from those found in a given chunk. The default+// implementation of this logic removes all unverified results if there are any verified results, and all unverified+// results except for one otherwise, but this interface allows a detector to specify different logic. (This logic must+// be implemented outside results generation because there are circumstances under which the engine should not execute+// it.)+type CustomResultsCleaner interface {+ // CleanResults removes "superfluous" results from a result set (where the definition of "superfluous" is detector-+ // specific).+ CleanResults(results []Result) []Result+ // ShouldCleanResultsIrrespectiveOfConfiguration allows a custom cleaner to instruct the engine to ignore+ // user-provided configuration that controls whether results are cleaned. (User-provided configuration is not the+ // only factor that determines whether the engine runs cleaning logic.)+ ShouldCleanResultsIrrespectiveOfConfiguration() bool+}+// Versioner is an optional interface that a detector can implement to// differentiate instances of the same detector type.type Versioner interface {diff --git a/pkg/detectors/opsgenie/opsgenie.go b/pkg/detectors/opsgenie/opsgenie.goindex 5f2938c13..ed0cf82bd 100644--- a/pkg/detectors/opsgenie/opsgenie.go+++ b/pkg/detectors/opsgenie/opsgenie.go@@ -94,5 +94,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultresults = append(results, s1)}- return detectors.CleanResults(results), nil+ return results, nil}diff --git a/pkg/detectors/razorpay/razorpay.go b/pkg/detectors/razorpay/razorpay.goindex 9f633f856..13a142478 100644--- a/pkg/detectors/razorpay/razorpay.go+++ b/pkg/detectors/razorpay/razorpay.go@@ -3,16 +3,17 @@ package razorpayimport ("context""encoding/json"- regexp "github.com/wasilibs/go-re2""io""net/http"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}@@ -77,7 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}- return detectors.CleanResults(results), nil+ return results, nil}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/detectors/twilio/twilio.go b/pkg/detectors/twilio/twilio.goindex 7c0b6fb5b..d0dd2e418 100644--- a/pkg/detectors/twilio/twilio.go+++ b/pkg/detectors/twilio/twilio.go@@ -114,7 +114,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}}- return detectors.CleanResults(results), nil+ return results, nil}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex dd051c596..4b524d45c 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -1060,15 +1060,24 @@ func (e *Engine) filterResults(detector *ahocorasick.DetectorMatch,results []detectors.Result,) []detectors.Result {- if e.filterUnverified {- results = detectors.CleanResults(results)+ clean := detectors.CleanResults+ ignoreConfig := false+ if cleaner, ok := detector.Detector.(detectors.CustomResultsCleaner); ok {+ clean = cleaner.CleanResults+ ignoreConfig = cleaner.ShouldCleanResultsIrrespectiveOfConfiguration()}+ if e.filterUnverified || ignoreConfig {+ results = clean(results)+ }+if !e.retainFalsePositives {results = detectors.FilterKnownFalsePositives(ctx, detector.Detector, results)}+if e.filterEntropy != 0 {results = detectors.FilterResultsWithEntropy(ctx, results, e.filterEntropy, e.retainFalsePositives)}+return results}diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.goindex b620c5fe5..c6ca39838 100644--- a/pkg/engine/engine_test.go+++ b/pkg/engine/engine_test.go@@ -841,6 +841,75 @@ func TestLikelyDuplicate(t *testing.T) {}}+type customCleaner struct {+ ignoreConfig bool+}++var _ detectors.CustomResultsCleaner = (*customCleaner)(nil)+var _ detectors.Detector = (*customCleaner)(nil)++func (c customCleaner) FromData(aCtx.Context, bool, []byte) ([]detectors.Result, error) {+ return []detectors.Result{}, nil+}++func (c customCleaner) Keywords() []string { return []string{} }+func (c customCleaner) Type() detectorspb.DetectorType { return detectorspb.DetectorType(-1) }++func (c customCleaner) CleanResults([]detectors.Result) []detectors.Result {+ return []detectors.Result{}+}+func (c customCleaner) ShouldCleanResultsIrrespectiveOfConfiguration() bool { return c.ignoreConfig }++func TestFilterResults_CustomCleaner(t *testing.T) {+ testCases := []struct {+ name string+ cleaningConfigured bool+ ignoreConfig bool+ resultsToClean []detectors.Result+ wantResults []detectors.Result+ }{+ {+ name: "respect config to clean",+ cleaningConfigured: true,+ ignoreConfig: false,+ resultsToClean: []detectors.Result{{}},+ wantResults: []detectors.Result{},+ },+ {+ name: "respect config to not clean",+ cleaningConfigured: false,+ ignoreConfig: false,+ resultsToClean: []detectors.Result{{}},+ wantResults: []detectors.Result{{}},+ },+ {+ name: "clean irrespective of config",+ cleaningConfigured: false,+ ignoreConfig: true,+ resultsToClean: []detectors.Result{{}},+ wantResults: []detectors.Result{},+ },+ }++ for _, tt := range testCases {+ t.Run(tt.name, func(t *testing.T) {+ match := ahocorasick.DetectorMatch{+ Detector: customCleaner{+ ignoreConfig: tt.ignoreConfig,+ },+ }+ engine := Engine{+ filterUnverified: tt.cleaningConfigured,+ retainFalsePositives: true,+ }++ cleaned := engine.filterResults(context.Background(), &match, tt.resultsToClean)++ assert.ElementsMatch(t, tt.wantResults, cleaned)+ })+ }+}+func BenchmarkPopulateMatchingDetectors(b *testing.B) {allDetectors := DefaultDetectors()ac := ahocorasick.NewAhoCorasickCore(allDetectors)