Skip to content

Find, verify, and analyze leaked credentials

Add ability to include and exclude detectors (#1106)

Miccahdd39848709f9
diff --git a/main.go b/main.go
index 85e344db4..13424f239 100644
--- a/main.go
+++ b/main.go
@@ -21,10 +21,12 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/config"
"github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/decoders"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine"
"github.com/trufflesecurity/trufflehog/v3/pkg/handlers"
"github.com/trufflesecurity/trufflehog/v3/pkg/log"
"github.com/trufflesecurity/trufflehog/v3/pkg/output"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources"
"github.com/trufflesecurity/trufflehog/v3/pkg/sources/git"
"github.com/trufflesecurity/trufflehog/v3/pkg/updater"
@@ -50,6 +52,8 @@ var (
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
+ includeDetectors = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()
+ excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()
gitScan = cli.Command("git", "Find credentials in git repositories.")
gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()
@@ -210,11 +214,40 @@ func run(state overseer.State) {
handlers.SetArchiveMaxTimeout(*archiveTimeout)
}
+ // Build include and exclude detector filter sets.
+ var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]struct{}
+ {
+ includeList, err := config.ParseDetectors(*includeDetectors)
+ if err != nil {
+ // Exit if there was an error to inform the user of the misconfiguration.
+ logger.Error(err, "invalid include list detector configuration")
+ os.Exit(1)
+ }
+ excludeList, err := config.ParseDetectors(*excludeDetectors)
+ if err != nil {
+ // Exit if there was an error to inform the user of the misconfiguration.
+ logger.Error(err, "invalid exclude list detector configuration")
+ os.Exit(1)
+ }
+ includeDetectorTypes = detectorTypeToSet(includeList)
+ excludeDetectorTypes = detectorTypeToSet(excludeList)
+ }
+ includeFilter := func(d detectors.Detector) bool {
+ _, ok := includeDetectorTypes[d.Type()]
+ return ok
+ }
+ excludeFilter := func(d detectors.Detector) bool {
+ _, ok := excludeDetectorTypes[d.Type()]
+ return !ok
+ }
+
e := engine.Start(ctx,
engine.WithConcurrency(*concurrency),
engine.WithDecoders(decoders.DefaultDecoders()...),
engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
engine.WithDetectors(!*noVerification, conf.Detectors...),
+ engine.WithFilterDetectors(includeFilter),
+ engine.WithFilterDetectors(excludeFilter),
engine.WithFilterUnverified(*filterUnverified),
)
@@ -396,3 +429,11 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
os.Exit(0)
}
}
+
+func detectorTypeToSet(detectors []detectorspb.DetectorType) map[detectorspb.DetectorType]struct{} {
+ output := make(map[detectorspb.DetectorType]struct{}, len(detectors))
+ for _, d := range detectors {
+ output[d] = struct{}{}
+ }
+ return output
+}
diff --git a/pkg/config/detectors.go b/pkg/config/detectors.go
new file mode 100644
index 000000000..b1ec0f471
--- /dev/null
+++ b/pkg/config/detectors.go
@@ -0,0 +1,140 @@
+package config
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+
+ dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "golang.org/x/exp/slices"
+)
+
+var (
+ specialGroups = map[string][]dpb.DetectorType{
+ "all": allDetectors(),
+ }
+
+ detectorTypeValue = make(map[string]dpb.DetectorType, len(dpb.DetectorType_value))
+ validDetectors = make(map[dpb.DetectorType]struct{}, len(dpb.DetectorType_value))
+ maxDetectorType dpb.DetectorType
+)
+
+// Setup package local global variables.
+func init() {
+ for k, v := range dpb.DetectorType_value {
+ dt := dpb.DetectorType(v)
+ detectorTypeValue[strings.ToLower(k)] = dt
+ validDetectors[dt] = struct{}{}
+ if dt > maxDetectorType {
+ maxDetectorType = dt
+ }
+ }
+}
+
+// ParseDetectors parses user supplied string into a list of detectors types.
+// "all" will return the list of all available detectors. The input is comma
+// separated and may use the case-insensitive detector name defined in the
+// protobuf, or the protobuf enum number. A range may be used as well in the
+// form "start-end". Order is preserved and duplicates are ignored.
+func ParseDetectors(input string) ([]dpb.DetectorType, error) {
+ var output []dpb.DetectorType
+ seenDetector := map[dpb.DetectorType]struct{}{}
+ for _, item := range strings.Split(input, ",") {
+ item = strings.TrimSpace(item)
+ if item == "" {
+ continue
+ }
+ detectors, ok := specialGroups[strings.ToLower(item)]
+ if !ok {
+ var err error
+ detectors, err = asRange(item)
+ if err != nil {
+ return nil, err
+ }
+ }
+ for _, d := range detectors {
+ if _, ok := seenDetector[d]; ok {
+ continue
+ }
+ seenDetector[d] = struct{}{}
+ output = append(output, d)
+ }
+ }
+ return output, nil
+}
+
+// allDetectors reutrns an ordered slice of all detector types.
+func allDetectors() []dpb.DetectorType {
+ all := make([]dpb.DetectorType, 0, len(dpb.DetectorType_name))
+ for id := range dpb.DetectorType_name {
+ all = append(all, dpb.DetectorType(id))
+ }
+ slices.Sort(all)
+ return all
+}
+
+// asRange converts a single input into a slice of detector types. If the input
+// is not in range format, a slice of length 1 is returned. Unbounded ranges
+// are allowed.
+func asRange(input string) ([]dpb.DetectorType, error) {
+ // Check if it's a single detector type.
+ dt, err := asDetectorType(input)
+ if err == nil {
+ return []dpb.DetectorType{dt}, nil
+ }
+
+ // Check if it's a range; if not return the error from above.
+ start, end, found := strings.Cut(input, "-")
+ if !found {
+ return nil, err
+ }
+ start, end = strings.TrimSpace(start), strings.TrimSpace(end)
+
+ // Convert the range start and end to a DetectorType.
+ dtStart, err := asDetectorType(start)
+ if err != nil {
+ return nil, err
+ }
+ dtEnd, err := asDetectorType(end)
+ // If end is empty it's an unbounded range.
+ if err != nil && end != "" {
+ return nil, err
+ }
+ if end == "" {
+ dtEnd = maxDetectorType
+ }
+
+ step := dpb.DetectorType(1)
+ if dtStart > dtEnd {
+ step = -1
+ }
+ var output []dpb.DetectorType
+ for dt := dtStart; dt != dtEnd; dt += step {
+ if _, ok := validDetectors[dt]; !ok {
+ continue
+ }
+ output = append(output, dt)
+ }
+ return append(output, dtEnd), nil
+}
+
+// asDetectorType converts the case-insensitive input into a detector type.
+// Name or ID may be used.
+func asDetectorType(input string) (dpb.DetectorType, error) {
+ if input == "" {
+ return 0, fmt.Errorf("empty detector")
+ }
+ // Check if it's a named detector.
+ if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {
+ return dt, nil
+ }
+ // Check if it's a detector ID.
+ if i, err := strconv.ParseInt(input, 10, 32); err == nil {
+ dt := dpb.DetectorType(i)
+ if _, ok := validDetectors[dt]; !ok {
+ return 0, fmt.Errorf("invalid detector ID: %s", input)
+ }
+ return dt, nil
+ }
+ return 0, fmt.Errorf("unrecognized detector type: %s", input)
+}
diff --git a/pkg/config/detectors_test.go b/pkg/config/detectors_test.go
new file mode 100644
index 000000000..422d3c2fe
--- /dev/null
+++ b/pkg/config/detectors_test.go
@@ -0,0 +1,40 @@
+package config
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestDetectorParsing(t *testing.T) {
+ tests := map[string]struct {
+ input string
+ expected []dpb.DetectorType
+ }{
+ "all": {"AlL", allDetectors()},
+ "trailing range": {"0-", allDetectors()},
+ "all after 1": {"1-", allDetectors()[1:]},
+ "named and valid range": {"aWs,8-9", []dpb.DetectorType{dpb.DetectorType_AWS, dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},
+ "duplicate order preserved": {"9, 8, 9", []dpb.DetectorType{9, 8}},
+ "named range": {"github - gitlab", []dpb.DetectorType{dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},
+ "range preserved": {"8-9, 7-10", []dpb.DetectorType{8, 9, 7, 10}},
+ "reverse range": {"9-8", []dpb.DetectorType{9, 8}},
+ "range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},
+ "empty list item": {"8, ,9", []dpb.DetectorType{8, 9}},
+ "invalid end range": {"0-1337", nil},
+ "invalid name": {"foo", nil},
+ "negative": {"-1", nil},
+ }
+
+ for name, tt := range tests {
+ t.Run(name, func(t *testing.T) {
+ got, gotErr := ParseDetectors(tt.input)
+ if tt.expected == nil {
+ assert.Error(t, gotErr)
+ return
+ }
+ assert.Equal(t, tt.expected, got)
+ })
+ }
+}
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 3b31a39e8..6b179adec 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -73,6 +73,31 @@ func WithFilterUnverified(filter bool) EngineOption {
}
}
+// WithFilterDetectors applies a filter to the configured list of detectors. If
+// the filterFunc returns true, the detector will be included for scanning.
+// This option applies to the existing list of detectors configured, so the
+// order this option appears matters. All filtering happens before scanning.
+func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {
+ return func(e *Engine) {
+ // If no detectors are configured, do nothing.
+ if e.detectors == nil {
+ return
+ }
+ e.detectors[true] = filterDetectors(filterFunc, e.detectors[true])
+ e.detectors[false] = filterDetectors(filterFunc, e.detectors[false])
+ }
+}
+
+func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {
+ var output []detectors.Detector
+ for _, detector := range input {
+ if filterFunc(detector) {
+ output = append(output, detector)
+ }
+ }
+ return output
+}
+
func Start(ctx context.Context, options ...EngineOption) *Engine {
e := &Engine{
chunks: make(chan *sources.Chunk),