Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Support filtering detectors by version (#1150)
Miccahe6846ede549d
diff --git a/main.go b/main.goindex 49707b946..acb75cb38 100644--- a/main.go+++ b/main.go@@ -216,30 +216,52 @@ func run(state overseer.State) {}// Build include and exclude detector filter sets.- var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]struct{}+ var includeDetectorTypes, excludeDetectorTypes map[detectorspb.DetectorType]config.DetectorID{includeList, err := config.ParseDetectors(*includeDetectors)if err != nil {// Exit if there was an error to inform the user of the misconfiguration.- logger.Error(err, "invalid include list detector configuration")- os.Exit(1)+ logFatal(err, "invalid include list detector configuration")}excludeList, err := config.ParseDetectors(*excludeDetectors)if err != nil {// Exit if there was an error to inform the user of the misconfiguration.- logger.Error(err, "invalid exclude list detector configuration")- os.Exit(1)+ logFatal(err, "invalid exclude list detector configuration")}- includeDetectorTypes = detectorTypeToSet(includeList)- excludeDetectorTypes = detectorTypeToSet(excludeList)+ includeDetectorTypes = detectorTypeToMap(includeList)+ excludeDetectorTypes = detectorTypeToMap(excludeList)}includeFilter := func(d detectors.Detector) bool {- _, ok := includeDetectorTypes[d.Type()]- return ok+ id, ok := includeDetectorTypes[d.Type()]+ if id.Version == 0 {+ return ok+ }+ versionD, ok := d.(detectors.Versioner)+ if !ok {+ // Error: version provided but not a detectors.Versioner+ logFatal(+ fmt.Errorf("version provided but detector does not have a version"),+ "invalid include list detector configuration",+ "detector", id,+ )+ }+ return versionD.Version() == id.Version}excludeFilter := func(d detectors.Detector) bool {- _, ok := excludeDetectorTypes[d.Type()]- return !ok+ id, ok := excludeDetectorTypes[d.Type()]+ if id.Version == 0 {+ return !ok+ }+ versionD, ok := d.(detectors.Versioner)+ if !ok {+ // Error: version provided but not a detectors.Versioner+ logFatal(+ fmt.Errorf("version provided but detector does not have a version"),+ "invalid exclude list detector configuration",+ "detector", id,+ )+ }+ return versionD.Version() != id.Version}e := engine.Start(ctx,@@ -431,10 +453,10 @@ func logFatalFunc(logger logr.Logger) func(error, string, ...any) {}}-func detectorTypeToSet(detectors []detectorspb.DetectorType) map[detectorspb.DetectorType]struct{} {- output := make(map[detectorspb.DetectorType]struct{}, len(detectors))+func detectorTypeToMap(detectors []config.DetectorID) map[detectorspb.DetectorType]config.DetectorID {+ output := make(map[detectorspb.DetectorType]config.DetectorID, len(detectors))for _, d := range detectors {- output[d] = struct{}{}+ output[d.ID] = d}return output}diff --git a/pkg/config/detectors.go b/pkg/config/detectors.goindex b1ec0f471..6ee31c61b 100644--- a/pkg/config/detectors.go+++ b/pkg/config/detectors.go@@ -2,15 +2,15 @@ package configimport ("fmt"+ "sort""strconv""strings"dpb "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"- "golang.org/x/exp/slices")var (- specialGroups = map[string][]dpb.DetectorType{+ specialGroups = map[string][]DetectorID{"all": allDetectors(),}@@ -31,14 +31,19 @@ func init() {}}+type DetectorID struct {+ ID dpb.DetectorType+ Version int+}+// ParseDetectors parses user supplied string into a list of detectors types.// "all" will return the list of all available detectors. The input is comma// separated and may use the case-insensitive detector name defined in the// protobuf, or the protobuf enum number. A range may be used as well in the// form "start-end". Order is preserved and duplicates are ignored.-func ParseDetectors(input string) ([]dpb.DetectorType, error) {- var output []dpb.DetectorType- seenDetector := map[dpb.DetectorType]struct{}{}+func ParseDetectors(input string) ([]DetectorID, error) {+ var output []DetectorID+ seenDetector := map[DetectorID]struct{}{}for _, item := range strings.Split(input, ",") {item = strings.TrimSpace(item)if item == "" {@@ -63,24 +68,35 @@ func ParseDetectors(input string) ([]dpb.DetectorType, error) {return output, nil}+func (id DetectorID) String() string {+ name := dpb.DetectorType_name[int32(id.ID)]+ if name == "" {+ name = "<invalid ID>"+ }+ if id.Version == 0 {+ return name+ }+ return fmt.Sprintf("%s.v%d", name, id.Version)+}+// allDetectors reutrns an ordered slice of all detector types.-func allDetectors() []dpb.DetectorType {- all := make([]dpb.DetectorType, 0, len(dpb.DetectorType_name))+func allDetectors() []DetectorID {+ all := make([]DetectorID, 0, len(dpb.DetectorType_name))for id := range dpb.DetectorType_name {- all = append(all, dpb.DetectorType(id))+ all = append(all, DetectorID{ID: dpb.DetectorType(id)})}- slices.Sort(all)+ sort.Slice(all, func(i, j int) bool { return all[i].ID < all[j].ID })return all}// asRange converts a single input into a slice of detector types. If the input// is not in range format, a slice of length 1 is returned. Unbounded ranges// are allowed.-func asRange(input string) ([]dpb.DetectorType, error) {+func asRange(input string) ([]DetectorID, error) {// Check if it's a single detector type.- dt, err := asDetectorType(input)+ dt, err := asDetectorID(input)if err == nil {- return []dpb.DetectorType{dt}, nil+ return []DetectorID{dt}, nil}// Check if it's a range; if not return the error from above.@@ -91,50 +107,77 @@ func asRange(input string) ([]dpb.DetectorType, error) {start, end = strings.TrimSpace(start), strings.TrimSpace(end)// Convert the range start and end to a DetectorType.- dtStart, err := asDetectorType(start)+ dtStart, err := asDetectorID(start)if err != nil {return nil, err}- dtEnd, err := asDetectorType(end)+ dtEnd, err := asDetectorID(end)// If end is empty it's an unbounded range.if err != nil && end != "" {return nil, err}if end == "" {- dtEnd = maxDetectorType+ dtEnd.ID = maxDetectorType+ }++ // Ensure these ranges don't have versions.+ if dtEnd.Version != 0 || dtStart.Version != 0 {+ return nil, fmt.Errorf("versions within ranges are not supported: %s", input)}step := dpb.DetectorType(1)- if dtStart > dtEnd {+ if dtStart.ID > dtEnd.ID {step = -1}- var output []dpb.DetectorType- for dt := dtStart; dt != dtEnd; dt += step {+ var output []DetectorID+ for dt := dtStart.ID; dt != dtEnd.ID; dt += step {if _, ok := validDetectors[dt]; !ok {continue}- output = append(output, dt)+ output = append(output, DetectorID{ID: dt})}return append(output, dtEnd), nil}-// asDetectorType converts the case-insensitive input into a detector type.+// asDetectorID converts the case-insensitive input into a DetectorID.// Name or ID may be used.-func asDetectorType(input string) (dpb.DetectorType, error) {+func asDetectorID(input string) (DetectorID, error) {if input == "" {- return 0, fmt.Errorf("empty detector")+ return DetectorID{}, fmt.Errorf("empty detector")+ }+ var detectorID DetectorID+ // Separate the version if there is one.+ if detector, version, hasVersion := strings.Cut(input, "."); hasVersion {+ parsedVersion, err := parseVersion(version)+ if err != nil {+ return DetectorID{}, fmt.Errorf("invalid version for input: %q error: %w", input, err)+ }+ detectorID.Version = parsedVersion+ // Because there was a version, the detector type input is the part before the '.'+ input = detector}+// Check if it's a named detector.if dt, ok := detectorTypeValue[strings.ToLower(input)]; ok {- return dt, nil+ detectorID.ID = dt+ return detectorID, nil}// Check if it's a detector ID.if i, err := strconv.ParseInt(input, 10, 32); err == nil {dt := dpb.DetectorType(i)if _, ok := validDetectors[dt]; !ok {- return 0, fmt.Errorf("invalid detector ID: %s", input)+ return DetectorID{}, fmt.Errorf("invalid detector ID: %s", input)}- return dt, nil+ detectorID.ID = dt+ return detectorID, nil+ }+ return DetectorID{}, fmt.Errorf("unrecognized detector type: %s", input)+}++func parseVersion(v string) (int, error) {+ if !strings.HasPrefix(strings.ToLower(v), "v") {+ return 0, fmt.Errorf("version must start with 'v'")}- return 0, fmt.Errorf("unrecognized detector type: %s", input)+ version := strings.TrimLeft(v, "vV")+ return strconv.Atoi(version)}diff --git a/pkg/config/detectors_test.go b/pkg/config/detectors_test.goindex 422d3c2fe..9b3d2ae76 100644--- a/pkg/config/detectors_test.go+++ b/pkg/config/detectors_test.go@@ -10,21 +10,28 @@ import (func TestDetectorParsing(t *testing.T) {tests := map[string]struct {input string- expected []dpb.DetectorType+ expected []DetectorID}{"all": {"AlL", allDetectors()},"trailing range": {"0-", allDetectors()},"all after 1": {"1-", allDetectors()[1:]},- "named and valid range": {"aWs,8-9", []dpb.DetectorType{dpb.DetectorType_AWS, dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},- "duplicate order preserved": {"9, 8, 9", []dpb.DetectorType{9, 8}},- "named range": {"github - gitlab", []dpb.DetectorType{dpb.DetectorType_Github, dpb.DetectorType_Gitlab}},- "range preserved": {"8-9, 7-10", []dpb.DetectorType{8, 9, 7, 10}},- "reverse range": {"9-8", []dpb.DetectorType{9, 8}},+ "named and valid range": {"aWs,8-9", []DetectorID{{ID: dpb.DetectorType_AWS}, {ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},+ "duplicate order preserved": {"9, 8, 9", []DetectorID{{ID: 9}, {ID: 8}}},+ "named range": {"github - gitlab", []DetectorID{{ID: dpb.DetectorType_Github}, {ID: dpb.DetectorType_Gitlab}}},+ "range preserved": {"8-9, 7-10", []DetectorID{{ID: 8}, {ID: 9}, {ID: 7}, {ID: 10}}},+ "reverse range": {"9-8", []DetectorID{{ID: 9}, {ID: 8}}},"range preserved with all": {"10-,all", append(allDetectors()[10:], allDetectors()[:10]...)},- "empty list item": {"8, ,9", []dpb.DetectorType{8, 9}},+ "empty list item": {"8, ,9", []DetectorID{{ID: 8}, {ID: 9}}},"invalid end range": {"0-1337", nil},"invalid name": {"foo", nil},"negative": {"-1", nil},+ "github.v1": {"github.v1", []DetectorID{{ID: dpb.DetectorType_Github, Version: 1}}},+ "gitlab.v100": {"gitlab.v100", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 100}}},+ "range with versions": {"github.v2 - gitlab.v1", nil},+ "invalid version no v": {"gitlab.2", nil},+ "invalid version no number": {"gitlab.github", nil},+ "capital V is fine": {"GiTlAb.V2", []DetectorID{{ID: dpb.DetectorType_Gitlab, Version: 2}}},+ "id number with version": {"8.v2", []DetectorID{{ID: 8, Version: 2}}},}for name, tt := range tests {diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex 454348048..3e376d19e 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -25,6 +25,12 @@ type Detector interface {Type() detectorspb.DetectorType}+// Versioner is an optional interface that a detector can implement to+// differentiate instances of the same detector type.+type Versioner interface {+ Version() int+}+type Result struct {// DetectorType is the type of Detector.DetectorType detectorspb.DetectorTypediff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.goindex 4c8e3f752..54abf074f 100644--- a/pkg/detectors/github/github.go+++ b/pkg/detectors/github/github.go@@ -15,8 +15,11 @@ import (type Scanner struct{}-// Ensure the Scanner satisfies the interface at compile time.+// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.Versioner = (*Scanner)(nil)++func (s Scanner) Version() int { return 2 }var (// Oauth tokendiff --git a/pkg/detectors/github_old/github_old.go b/pkg/detectors/github_old/github_old.goindex 241b454ca..2d468d931 100644--- a/pkg/detectors/github_old/github_old.go+++ b/pkg/detectors/github_old/github_old.go@@ -14,8 +14,11 @@ import (type Scanner struct{}-// Ensure the Scanner satisfies the interface at compile time.+// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.Versioner = (*Scanner)(nil)++func (s Scanner) Version() int { return 1 }var (// Oauth token@@ -38,7 +41,7 @@ type userRes struct {// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {- return []string{"github","gh","pat"}+ return []string{"github", "gh", "pat"}}// FromData will find and optionally verify GitHub secrets in a given set of bytes.diff --git a/pkg/detectors/gitlab/gitlab.go b/pkg/detectors/gitlab/gitlab.goindex 82c1cb2a2..4ed804ec1 100644--- a/pkg/detectors/gitlab/gitlab.go+++ b/pkg/detectors/gitlab/gitlab.go@@ -14,8 +14,11 @@ import (type Scanner struct{}-// Ensure the Scanner satisfies the interface at compile time.+// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.Versioner = (*Scanner)(nil)++func (s Scanner) Version() int { return 1 }var (keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"gitlab"}) + `\b((?:glpat|)[a-zA-Z0-9\-=_]{20,22})\b`)diff --git a/pkg/detectors/gitlabv2/gitlab.go b/pkg/detectors/gitlabv2/gitlab.goindex 8453ab539..01ea72228 100644--- a/pkg/detectors/gitlabv2/gitlab.go+++ b/pkg/detectors/gitlabv2/gitlab.go@@ -13,8 +13,11 @@ import (type Scanner struct{}-// Ensure the Scanner satisfies the interface at compile time.+// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.Versioner = (*Scanner)(nil)++func (*Scanner) Version() int { return 2 }var (keyPat = regexp.MustCompile(`\b(glpat-[a-zA-Z0-9\-=_]{20,22})\b`)diff --git a/pkg/detectors/npmtoken/npmtoken.go b/pkg/detectors/npmtoken/npmtoken.goindex 30dfbb90e..351209d86 100644--- a/pkg/detectors/npmtoken/npmtoken.go+++ b/pkg/detectors/npmtoken/npmtoken.go@@ -14,8 +14,11 @@ import (type Scanner struct{}-// Ensure the Scanner satisfies the interface at compile time.+// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.Versioner = (*Scanner)(nil)++func (s Scanner) Version() int { return 1 }var (client = common.SaneHttpClient()@@ -34,7 +37,7 @@ func (s Scanner) Keywords() []string {func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)- matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ matches := keyPat.FindAllStringSubmatch(dataStr, -1)for _, match := range matches {if len(match) != 2 {continuediff --git a/pkg/detectors/npmtokenv2/npmtokenv2.go b/pkg/detectors/npmtokenv2/npmtokenv2.goindex db9761076..40e2fdf6a 100644--- a/pkg/detectors/npmtokenv2/npmtokenv2.go+++ b/pkg/detectors/npmtokenv2/npmtokenv2.go@@ -13,8 +13,11 @@ import (type Scanner struct{}-// Ensure the Scanner satisfies the interface at compile time.+// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.Versioner = (*Scanner)(nil)++func (s Scanner) Version() int { return 2 }var (client = common.SaneHttpClient()