Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add ability to include and exclude detectors (#1106)
Miccahdd39848709f9
diff --git a/main.go b/main.goindex 85e344db4..13424f239 100644--- a/main.go+++ b/main.go@@ -21,10 +21,12 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/config""github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/decoders"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/engine""github.com/trufflesecurity/trufflehog/v3/pkg/handlers""github.com/trufflesecurity/trufflehog/v3/pkg/log""github.com/trufflesecurity/trufflehog/v3/pkg/output"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb""github.com/trufflesecurity/trufflehog/v3/pkg/sources""github.com/trufflesecurity/trufflehog/v3/pkg/sources/git""github.com/trufflesecurity/trufflehog/v3/pkg/updater"@@ -50,6 +52,8 @@ var (archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()+ includeDetectors = cli.Flag("include-detectors", "Comma separated list of detector types to include. Protobuf name or IDs may be used, as well as ranges.").Default("all").String()+ excludeDetectors = cli.Flag("exclude-detectors", "Comma separated list of detector types to exclude. Protobuf name or IDs may be used, as well as ranges. IDs defined here take precedence over the include list.").String()gitScan = cli.Command("git", "Find credentials in git repositories.")gitScanURI = gitScan.Arg("uri", "Git repository URL. https://, file://, or ssh:// schema expected.").Required().String()@@ -210,11 +214,40 @@ func run(state overseer.State) {handlers.SetArchiveMaxTimeout(*archiveTimeout)}+ // Build include and exclude detector filter sets.+ var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]struct{}+ {+ includeList, err := config.ParseDetectors(*includeDetectors)+ if err != nil {+ // Exit if there was an error to inform the user of the misconfiguration.+ logger.Error(err, "invalid include list detector configuration")+ os.Exit(1)+ }+ excludeList, err := config.ParseDetectors(*excludeDetectors)+ if err != nil {+ // Exit if there was an error to inform the user of the misconfiguration.+ logger.Error(err, "invalid exclude list detector configuration")+ os.Exit(1)+ }+ includeDetectorTypes = detectorTypeToSet(includeList)+ excludeDetectorTypes = detectorTypeToSet(excludeList)+ }+ includeFilter := func(d detectors.Detector) bool {+ _, ok := includeDetectorTypes[d.Type()]+ return ok+ }+ excludeFilter := func(d detectors.Detector) bool {+ _, ok := excludeDetectorTypes[d.Type()]+ return !ok+ }+e := engine.Start(ctx,engine.WithConcurrency(*concurrency),engine.WithDecoders(decoders.DefaultDecoders()...),engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),engine.WithDetectors(!*noVerification, conf.Detectors...),+ engine.WithFilterDetectors(includeFilter),+ engine.WithFilterDetectors(excludeFilter),engine.WithFilterUnverified(*filterUnverified),)@@ -396,3 +429,11 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {os.Exit(0)}}++func detectorTypeToSet(detectors []detectorspb.DetectorType) map[detectorspb.DetectorType]struct{} {+ output := make(map[detectorspb.DetectorType]struct{}, len(detectors))+ for _, d := range detectors {+ output[d] = struct{}{}+ }+ return output+}diff --git a/pkg/config/detectors.go b/pkg/config/detectors.gonew file mode 100644index 000000000..b1ec0f471--- /dev/null+++ b/pkg/config/detectors.go@@ -0,0 +1,140 @@+package config++import (+ "fmt"+ "strconv"+ "strings"++ dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "golang.org/x/exp/slices"+)++var (+ specialGroups = map[string][]dpb.DetectorType{+ "all": allDetectors(),+ }++ detectorTypeValue = make(map[string]dpb.DetectorType, len(dpb.DetectorType_value))+ validDetectors = make(map[dpb.DetectorType]struct{}, len(dpb.DetectorType_value))+ maxDetectorType dpb.DetectorType+)++// Setup package local global variables.+func init() {+ for k, v := range dpb.DetectorType_value {+ dt := dpb.DetectorType(v)+ detectorTypeValue[strings.ToLower(k)] = dt+ validDetectors[dt] = struct{}{}+ if dt > maxDetectorType {+ maxDetectorType = dt+ }+ }+}++// ParseDetectors parses user supplied string into a list of detectors types.+// "all" will return the list of all available detectors. The input is comma+// separated and may use the case-insensitive detector name defined in the+// protobuf, or the protobuf enum number. A range may be used as well in the+// form "start-end". Order is preserved and duplicates are ignored.+func ParseDetectors(input string) ([]dpb.DetectorType, error) {+ var output []dpb.DetectorType+ seenDetector := map[dpb.DetectorType]struct{}{}+ for _, item := range strings.Split(input, ",") {+ item = strings.TrimSpace(item)+ if item == "" {+ continue+ }+ detectors, ok := specialGroups[strings.ToLower(item)]+ if !ok {+ var err error+ detectors, err = asRange(item)+ if err != nil {+ return nil, err+ }+ }+ for _, d := range detectors {+ if _, ok := seenDetector[d]; ok {+ continue+ }+ seenDetector[d] = struct{}{}+ output = append(output, d)+ }+ }+ return output, nil+}++// allDetectors reutrns an ordered slice of all detector types.+func allDetectors() []dpb.DetectorType {+ all := make([]dpb.DetectorType, 0, len(dpb.DetectorType_name))+ for id := range dpb.DetectorType_name {+ all = append(all, dpb.DetectorType(id))+ }+ slices.Sort(all)+ return all+}++// asRange converts a single input into a slice of detector types. If the input+// is not in range format, a slice of length 1 is returned. Unbounded ranges+// are allowed.+func asRange(input string) ([]dpb.DetectorType, error) {+ // Check if it's a single detector type.+ dt, err := asDetectorType(input)+ if err == nil {+ return []dpb.DetectorType{dt}, nil+ }++ // Check if it's a range; if not return the error from above.+ start, end, found := strings.Cut(input, "-")+ if !found {+ return nil, err+ }+ start, end = strings.TrimSpace(start), strings.TrimSpace(end)++ // Convert the range start and end to a DetectorType.+ dtStart, err := asDetectorType(start)+ if err != nil {+ return nil, err+ }+ dtEnd, err := asDetectorType(end)+ // If end is empty it's an unbounded range.+ if err != nil && end != "" {+ return nil, err+ }+ if end == "" {+ dtEnd = maxDetectorType+ }++ step := dpb.DetectorType(1)+ if dtStart > dtEnd {+ step = -1+ }+ var output []dpb.DetectorType+ for dt := dtStart; dt != dtEnd; dt += step {+ if _, ok := validDetectors[dt]; !ok {+ continue+ }+ output = append(output, dt)+ }+ return append(output, dtEnd), nil+}++// asDetectorType converts the case-insensitive input into a detector type.+// Name or ID may be used.+func asDetectorType(input string) (dpb.DetectorType, error) {+ if input == "" {+ return 0, fmt.Errorf("empty detector")+ }+ // Check if it's a named detector.+ if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {+ return dt, nil+ }+ // Check if it's a detector ID.+ if i, err := strconv.ParseInt(input, 10, 32); err == nil {+ dt := dpb.DetectorType(i)+ if _, ok := validDetectors[dt]; !ok {+ return 0, fmt.Errorf("invalid detector ID: %s", input)+ }+ return dt, nil+ }+ return 0, fmt.Errorf("unrecognized detector type: %s", input)+}diff --git a/pkg/config/detectors_test.go b/pkg/config/detectors_test.gonew file mode 100644index 000000000..422d3c2fe--- /dev/null+++ b/pkg/config/detectors_test.go@@ -0,0 +1,40 @@+package config++import (+ "testing"++ "github.com/stretchr/testify/assert"+ dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestDetectorParsing(t *testing.T) {+ tests := map[string]struct {+ input string+ expected []dpb.DetectorType+ }{+ "all": {"AlL", allDetectors()},+ "trailing range": {"0-", allDetectors()},+ "all after 1": {"1-", allDetectors()[1:]},+ "named and valid range": {"aWs,8-9", []dpb.DetectorType{dpb.DetectorType_AWS, dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},+ "duplicate order preserved": {"9, 8, 9", []dpb.DetectorType{9, 8}},+ "named range": {"github - gitlab", []dpb.DetectorType{dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},+ "range preserved": {"8-9, 7-10", []dpb.DetectorType{8, 9, 7, 10}},+ "reverse range": {"9-8", []dpb.DetectorType{9, 8}},+ "range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},+ "empty list item": {"8, ,9", []dpb.DetectorType{8, 9}},+ "invalid end range": {"0-1337", nil},+ "invalid name": {"foo", nil},+ "negative": {"-1", nil},+ }++ for name, tt := range tests {+ t.Run(name, func(t *testing.T) {+ got, gotErr := ParseDetectors(tt.input)+ if tt.expected == nil {+ assert.Error(t, gotErr)+ return+ }+ assert.Equal(t, tt.expected, got)+ })+ }+}diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 3b31a39e8..6b179adec 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -73,6 +73,31 @@ func WithFilterUnverified(filter bool) EngineOption {}}+// WithFilterDetectors applies a filter to the configured list of detectors. If+// the filterFunc returns true, the detector will be included for scanning.+// This option applies to the existing list of detectors configured, so the+// order this option appears matters. All filtering happens before scanning.+func WithFilterDetectors(filterFunc func(detectors.Detector) bool) EngineOption {+ return func(e *Engine) {+ // If no detectors are configured, do nothing.+ if e.detectors == nil {+ return+ }+ e.detectors[true] = filterDetectors(filterFunc, e.detectors[true])+ e.detectors[false] = filterDetectors(filterFunc, e.detectors[false])+ }+}++func filterDetectors(filterFunc func(detectors.Detector) bool, input []detectors.Detector) []detectors.Detector {+ var output []detectors.Detector+ for _, detector := range input {+ if filterFunc(detector) {+ output = append(output, detector)+ }+ }+ return output+}+func Start(ctx context.Context, options ...EngineOption) *Engine {e := &Engine{chunks: make(chan *sources.Chunk),