Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
updated github v1 detector to ignore matches with known common prefixes (#4329)
Kashif Khand7e295aa4938
diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.goindex e61749f78..337b21ffa 100644--- a/pkg/detectors/github/v1/github_old.go+++ b/pkg/detectors/github/v1/github_old.go@@ -5,6 +5,7 @@ import ("encoding/json""fmt""net/http"+ "strings"regexp "github.com/wasilibs/go-re2"@@ -27,7 +28,8 @@ func (Scanner) CloudEndpoint() string { return "https://api.github.com" }var (// Oauth token// https://developer.github.com/v3/#oauth2-token-sent-in-a-header- keyPat = regexp.MustCompile(`(?i)(?:github|gh|pat|token)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)+ // the middle regex `\b[a-zA-Z0-9.\/?=&]{0,40}` is to match the prefix of token match to avoid processing common known patterns+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b[a-zA-Z0-9.\/?=&]{0,40}` + `\b([a-f0-9]{40})\b`)// TODO: Oauth2 client_id and client_secret// https://developer.github.com/v3/#oauth2-keysecret@@ -55,13 +57,25 @@ type HeaderInfo struct {// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {- return []string{"github", "gh", "pat", "token"}+ return []string{"github", "gh"}+}++func (s Scanner) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType_Github+}++func (s Scanner) Description() string {+ return "GitHub is a web-based platform used for version control and collaborative software development. GitHub tokens can be used to access and modify repositories and other resources."}var ghFalsePositives = map[detectors.FalsePositive]struct{}{detectors.FalsePositive("github commit"): {},}+var ghKnownNonSensitivePrefixes = []string{+ "avatars.githubusercontent.com", // github avatar urls prefix+}+// FromData will find and optionally verify GitHub secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)@@ -70,7 +84,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultfor _, match := range matches {// First match is entire regex, second is the first group.-+ matchPrefix := match[0]token := match[1]// Note that this false positive check happens **before** verification! I don't know why it's written this way@@ -79,6 +93,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultcontinue}+ // to avoid false positives+ if isKnownNonSensitiveCommonPrefix(matchPrefix) {+ continue+ }++ if detectors.StringShannonEntropy(token) < 3.5 {+ continue+ }+s1 := detectors.Result{DetectorType: detectorspb.DetectorType_Github,Raw: []byte(token),@@ -180,10 +203,16 @@ func SetHeaderInfo(headers *HeaderInfo, s1 *detectors.Result) {}}-func (s Scanner) Type() detectorspb.DetectorType {- return detectorspb.DetectorType_Github-}+// isKnownNonSensitiveCommonPrefix checks if the given prefix is a known, non-sensitive value.+// The GitHub v1 detector uses a broad regex that can capture many false positives.+// This function helps filter out matches that begin with common, safe prefixes.+// Example: avatars.githubusercontent.com/u/56769451?u=088102b6160822bc68c25a2a5df170080d0b16a2+func isKnownNonSensitiveCommonPrefix(matchPrefix string) bool {+ for _, prefix := range ghKnownNonSensitivePrefixes {+ if strings.Contains(matchPrefix, prefix) {+ return true+ }+ }-func (s Scanner) Description() string {- return "GitHub is a web-based platform used for version control and collaborative software development. GitHub tokens can be used to access and modify repositories and other resources."+ return false}