Skip to content

Find, verify, and analyze leaked credentials

Customize results cleaning (using smuggled interface) (#3235)

Cody Rosef39a5254ffd4
diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.go
index 2527e9236..0416f051e 100644
--- a/pkg/detectors/aws/aws.go
+++ b/pkg/detectors/aws/aws.go
@@ -105,6 +105,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
// Ensure the scanner satisfies the interface at compile time.
var _ detectors.Detector = (*scanner)(nil)
var _ detectors.MultiPartCredentialProvider = (*scanner)(nil)
+var _ detectors.CustomResultsCleaner = (*scanner)(nil)
var (
defaultVerificationClient = common.SaneHttpClient()
@@ -245,7 +246,11 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
}
}
- return awsCustomCleanResults(results), nil
+ return results, nil
+}
+
+func (s scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool {
+ return true
}
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
@@ -399,7 +404,7 @@ func (s scanner) verifyCanary(resIDMatch, resSecretMatch string) (bool, string,
}
}
-func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
+func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {
if len(results) == 0 {
return results
}
diff --git a/pkg/detectors/awssessionkeys/awssessionkey.go b/pkg/detectors/awssessionkeys/awssessionkey.go
index 4974f3d76..4cd0afdba 100644
--- a/pkg/detectors/awssessionkeys/awssessionkey.go
+++ b/pkg/detectors/awssessionkeys/awssessionkey.go
@@ -49,6 +49,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
// Ensure the scanner satisfies the interface at compile time.
var _ detectors.Detector = (*scanner)(nil)
+var _ detectors.CustomResultsCleaner = (*scanner)(nil)
var (
defaultVerificationClient = common.SaneHttpClient()
@@ -167,7 +168,11 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
}
}
- return awsCustomCleanResults(results), nil
+ return results, nil
+}
+
+func (s scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool {
+ return true
}
func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, resSessionMatch string, retryOn403 bool) (bool, map[string]string, error) {
@@ -283,7 +288,7 @@ func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch str
}
}
-func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
+func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {
if len(results) == 0 {
return results
}
diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go
index b3e975ff8..b4f643f97 100644
--- a/pkg/detectors/detectors.go
+++ b/pkg/detectors/detectors.go
@@ -26,6 +26,22 @@ type Detector interface {
Type() detectorspb.DetectorType
}
+// CustomResultsCleaner is an optional interface that a detector can implement to customize how its generated results
+// are "cleaned," which is defined as removing superfluous results from those found in a given chunk. The default
+// implementation of this logic removes all unverified results if there are any verified results, and all unverified
+// results except for one otherwise, but this interface allows a detector to specify different logic. (This logic must
+// be implemented outside results generation because there are circumstances under which the engine should not execute
+// it.)
+type CustomResultsCleaner interface {
+ // CleanResults removes "superfluous" results from a result set (where the definition of "superfluous" is detector-
+ // specific).
+ CleanResults(results []Result) []Result
+ // ShouldCleanResultsIrrespectiveOfConfiguration allows a custom cleaner to instruct the engine to ignore
+ // user-provided configuration that controls whether results are cleaned. (User-provided configuration is not the
+ // only factor that determines whether the engine runs cleaning logic.)
+ ShouldCleanResultsIrrespectiveOfConfiguration() bool
+}
+
// Versioner is an optional interface that a detector can implement to
// differentiate instances of the same detector type.
type Versioner interface {
diff --git a/pkg/detectors/opsgenie/opsgenie.go b/pkg/detectors/opsgenie/opsgenie.go
index 5f2938c13..ed0cf82bd 100644
--- a/pkg/detectors/opsgenie/opsgenie.go
+++ b/pkg/detectors/opsgenie/opsgenie.go
@@ -94,5 +94,5 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
results = append(results, s1)
}
- return detectors.CleanResults(results), nil
+ return results, nil
}
diff --git a/pkg/detectors/razorpay/razorpay.go b/pkg/detectors/razorpay/razorpay.go
index 9f633f856..13a142478 100644
--- a/pkg/detectors/razorpay/razorpay.go
+++ b/pkg/detectors/razorpay/razorpay.go
@@ -3,16 +3,17 @@ package razorpay
import (
"context"
"encoding/json"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{
+type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
}
@@ -77,7 +78,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
- return detectors.CleanResults(results), nil
+ return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
diff --git a/pkg/detectors/twilio/twilio.go b/pkg/detectors/twilio/twilio.go
index 7c0b6fb5b..d0dd2e418 100644
--- a/pkg/detectors/twilio/twilio.go
+++ b/pkg/detectors/twilio/twilio.go
@@ -114,7 +114,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
}
- return detectors.CleanResults(results), nil
+ return results, nil
}
func (s Scanner) Type() detectorspb.DetectorType {
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index dd051c596..4b524d45c 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -1060,15 +1060,24 @@ func (e *Engine) filterResults(
detector *ahocorasick.DetectorMatch,
results []detectors.Result,
) []detectors.Result {
- if e.filterUnverified {
- results = detectors.CleanResults(results)
+ clean := detectors.CleanResults
+ ignoreConfig := false
+ if cleaner, ok := detector.Detector.(detectors.CustomResultsCleaner); ok {
+ clean = cleaner.CleanResults
+ ignoreConfig = cleaner.ShouldCleanResultsIrrespectiveOfConfiguration()
}
+ if e.filterUnverified || ignoreConfig {
+ results = clean(results)
+ }
+
if !e.retainFalsePositives {
results = detectors.FilterKnownFalsePositives(ctx, detector.Detector, results)
}
+
if e.filterEntropy != 0 {
results = detectors.FilterResultsWithEntropy(ctx, results, e.filterEntropy, e.retainFalsePositives)
}
+
return results
}
diff --git a/pkg/engine/engine_test.go b/pkg/engine/engine_test.go
index b620c5fe5..c6ca39838 100644
--- a/pkg/engine/engine_test.go
+++ b/pkg/engine/engine_test.go
@@ -841,6 +841,75 @@ func TestLikelyDuplicate(t *testing.T) {
}
}
+type customCleaner struct {
+ ignoreConfig bool
+}
+
+var _ detectors.CustomResultsCleaner = (*customCleaner)(nil)
+var _ detectors.Detector = (*customCleaner)(nil)
+
+func (c customCleaner) FromData(aCtx.Context, bool, []byte) ([]detectors.Result, error) {
+ return []detectors.Result{}, nil
+}
+
+func (c customCleaner) Keywords() []string { return []string{} }
+func (c customCleaner) Type() detectorspb.DetectorType { return detectorspb.DetectorType(-1) }
+
+func (c customCleaner) CleanResults([]detectors.Result) []detectors.Result {
+ return []detectors.Result{}
+}
+func (c customCleaner) ShouldCleanResultsIrrespectiveOfConfiguration() bool { return c.ignoreConfig }
+
+func TestFilterResults_CustomCleaner(t *testing.T) {
+ testCases := []struct {
+ name string
+ cleaningConfigured bool
+ ignoreConfig bool
+ resultsToClean []detectors.Result
+ wantResults []detectors.Result
+ }{
+ {
+ name: "respect config to clean",
+ cleaningConfigured: true,
+ ignoreConfig: false,
+ resultsToClean: []detectors.Result{{}},
+ wantResults: []detectors.Result{},
+ },
+ {
+ name: "respect config to not clean",
+ cleaningConfigured: false,
+ ignoreConfig: false,
+ resultsToClean: []detectors.Result{{}},
+ wantResults: []detectors.Result{{}},
+ },
+ {
+ name: "clean irrespective of config",
+ cleaningConfigured: false,
+ ignoreConfig: true,
+ resultsToClean: []detectors.Result{{}},
+ wantResults: []detectors.Result{},
+ },
+ }
+
+ for _, tt := range testCases {
+ t.Run(tt.name, func(t *testing.T) {
+ match := ahocorasick.DetectorMatch{
+ Detector: customCleaner{
+ ignoreConfig: tt.ignoreConfig,
+ },
+ }
+ engine := Engine{
+ filterUnverified: tt.cleaningConfigured,
+ retainFalsePositives: true,
+ }
+
+ cleaned := engine.filterResults(context.Background(), &match, tt.resultsToClean)
+
+ assert.ElementsMatch(t, tt.wantResults, cleaned)
+ })
+ }
+}
+
func BenchmarkPopulateMatchingDetectors(b *testing.B) {
allDetectors := DefaultDetectors()
ac := ahocorasick.NewAhoCorasickCore(allDetectors)