Skip to content

Find, verify, and analyze leaked credentials

updated github v1 detector to ignore matches with known common prefixes (#4329)

Kashif Khand7e295aa4938
diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.go
index e61749f78..337b21ffa 100644
--- a/pkg/detectors/github/v1/github_old.go
+++ b/pkg/detectors/github/v1/github_old.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"net/http"
+ "strings"
regexp "github.com/wasilibs/go-re2"
@@ -27,7 +28,8 @@ func (Scanner) CloudEndpoint() string { return "https://api.github.com" }
var (
// Oauth token
// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
- keyPat = regexp.MustCompile(`(?i)(?:github|gh|pat|token)[^\.].{0,40}[ =:'"]+([a-f0-9]{40})\b`)
+ // the middle regex `\b[a-zA-Z0-9.\/?=&]{0,40}` is to match the prefix of token match to avoid processing common known patterns
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b[a-zA-Z0-9.\/?=&]{0,40}` + `\b([a-f0-9]{40})\b`)
// TODO: Oauth2 client_id and client_secret
// https://developer.github.com/v3/#oauth2-keysecret
@@ -55,13 +57,25 @@ type HeaderInfo struct {
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
- return []string{"github", "gh", "pat", "token"}
+ return []string{"github", "gh"}
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+ return detectorspb.DetectorType_Github
+}
+
+func (s Scanner) Description() string {
+ return "GitHub is a web-based platform used for version control and collaborative software development. GitHub tokens can be used to access and modify repositories and other resources."
}
var ghFalsePositives = map[detectors.FalsePositive]struct{}{
detectors.FalsePositive("github commit"): {},
}
+var ghKnownNonSensitivePrefixes = []string{
+ "avatars.githubusercontent.com", // github avatar urls prefix
+}
+
// FromData will find and optionally verify GitHub secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
@@ -70,7 +84,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
for _, match := range matches {
// First match is entire regex, second is the first group.
-
+ matchPrefix := match[0]
token := match[1]
// Note that this false positive check happens **before** verification! I don't know why it's written this way
@@ -79,6 +93,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
+ // to avoid false positives
+ if isKnownNonSensitiveCommonPrefix(matchPrefix) {
+ continue
+ }
+
+ if detectors.StringShannonEntropy(token) < 3.5 {
+ continue
+ }
+
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Github,
Raw: []byte(token),
@@ -180,10 +203,16 @@ func SetHeaderInfo(headers *HeaderInfo, s1 *detectors.Result) {
}
}
-func (s Scanner) Type() detectorspb.DetectorType {
- return detectorspb.DetectorType_Github
-}
+// isKnownNonSensitiveCommonPrefix checks if the given prefix is a known, non-sensitive value.
+// The GitHub v1 detector uses a broad regex that can capture many false positives.
+// This function helps filter out matches that begin with common, safe prefixes.
+// Example: avatars.githubusercontent.com/u/56769451?u=088102b6160822bc68c25a2a5df170080d0b16a2
+func isKnownNonSensitiveCommonPrefix(matchPrefix string) bool {
+ for _, prefix := range ghKnownNonSensitivePrefixes {
+ if strings.Contains(matchPrefix, prefix) {
+ return true
+ }
+ }
-func (s Scanner) Description() string {
- return "GitHub is a web-based platform used for version control and collaborative software development. GitHub tokens can be used to access and modify repositories and other resources."
+ return false
}