Skip to content

Find, verify, and analyze leaked credentials

detectors/github/v2: differentiate token type in ExtraData (#5223)

Shawn Hartsock8ee38640f6e6
diff --git a/pkg/detectors/github/v2/github.go b/pkg/detectors/github/v2/github.go
index bd3581237..e20ef6c28 100644
--- a/pkg/detectors/github/v2/github.go
+++ b/pkg/detectors/github/v2/github.go
@@ -3,6 +3,7 @@ package github
import (
"context"
"fmt"
+ "strings"
regexp "github.com/wasilibs/go-re2"
@@ -37,8 +38,34 @@ var (
// TODO: Oauth2 client_id and client_secret
// https://developer.github.com/v3/#oauth2-keysecret
+
+ // tokenTypesByPrefix maps each GitHub token prefix to a human-readable type.
+ // Each prefix identifies a materially different credential (PAT, OAuth
+ // grant, or GitHub App token) that is revoked/rotated through a different
+ // GitHub settings page.
+ // https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/
+ tokenTypesByPrefix = map[string]string{
+ "ghp_": "Personal Access Token (classic)",
+ "github_pat_": "Personal Access Token (fine-grained)",
+ "gho_": "OAuth Access Token",
+ "ghu_": "GitHub App User-to-Server Token",
+ "ghs_": "GitHub App Server-to-Server (installation) Token",
+ "ghr_": "GitHub App Refresh Token",
+ }
)
+// githubTokenType returns the human-readable token type for a matched token,
+// keyed off its prefix. Falls back to a generic label if no known prefix
+// matches (should not happen given keyPat, but keeps this safe).
+func githubTokenType(token string) string {
+ for prefix, tokenType := range tokenTypesByPrefix {
+ if strings.HasPrefix(token, prefix) {
+ return tokenType
+ }
+ }
+ return "Unknown GitHub token"
+}
+
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
@@ -62,6 +89,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"version": fmt.Sprintf("%d", s.Version()),
+ "token_type": githubTokenType(token),
},
SecretParts: map[string]string{"key": token},
}
diff --git a/pkg/detectors/github/v2/github_integration_test.go b/pkg/detectors/github/v2/github_integration_test.go
index 61861f569..20715b6a7 100644
--- a/pkg/detectors/github/v2/github_integration_test.go
+++ b/pkg/detectors/github/v2/github_integration_test.go
@@ -62,9 +62,10 @@ func TestGitHub_FromChunk(t *testing.T) {
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
"scopes": "notifications",
// "site_admin": "false", // not present in test verifiedGhp
- "url": "https://github.com/truffle-sandbox",
- "username": "truffle-sandbox",
- "version": "2",
+ "token_type": "Personal Access Token (classic)",
+ "url": "https://github.com/truffle-sandbox",
+ "username": "truffle-sandbox",
+ "version": "2",
},
},
},
@@ -84,6 +85,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+ "token_type": "Personal Access Token (classic)",
"version": "2",
},
},
@@ -104,6 +106,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+ "token_type": "OAuth Access Token",
"version": "2",
},
},
@@ -124,6 +127,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+ "token_type": "GitHub App User-to-Server Token",
"version": "2",
},
},
@@ -144,6 +148,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+ "token_type": "GitHub App Server-to-Server (installation) Token",
"version": "2",
},
},
@@ -164,6 +169,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+ "token_type": "GitHub App Refresh Token",
"version": "2",
},
},
@@ -184,6 +190,7 @@ func TestGitHub_FromChunk(t *testing.T) {
Verified: false,
ExtraData: map[string]string{
"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+ "token_type": "Personal Access Token (classic)",
"version": "2",
},
},
diff --git a/pkg/detectors/github/v2/github_test.go b/pkg/detectors/github/v2/github_test.go
index cc769e82a..263eed832 100644
--- a/pkg/detectors/github/v2/github_test.go
+++ b/pkg/detectors/github/v2/github_test.go
@@ -2,8 +2,12 @@ package github
import (
"context"
+ "reflect"
+ "strings"
"testing"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/google/go-cmp/cmp"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
@@ -86,3 +90,175 @@ func TestGithub_Pattern(t *testing.T) {
})
}
}
+
+func TestGithubTokenType(t *testing.T) {
+ tests := []struct {
+ prefix string
+ token string
+ want string
+ }{
+ {"ghp_", "ghp_" + strings.Repeat("a", 36), "Personal Access Token (classic)"},
+ {"github_pat_", "github_pat_" + strings.Repeat("a", 36), "Personal Access Token (fine-grained)"},
+ {"gho_", "gho_" + strings.Repeat("a", 36), "OAuth Access Token"},
+ {"ghu_", "ghu_" + strings.Repeat("a", 36), "GitHub App User-to-Server Token"},
+ {"ghs_", "ghs_" + strings.Repeat("a", 36), "GitHub App Server-to-Server (installation) Token"},
+ {"ghr_", "ghr_" + strings.Repeat("a", 36), "GitHub App Refresh Token"},
+ {"unknown", "not_a_github_token", "Unknown GitHub token"},
+ }
+
+ for _, test := range tests {
+ t.Run(test.prefix, func(t *testing.T) {
+ got := githubTokenType(test.token)
+ if got != test.want {
+ t.Errorf("githubTokenType(%q) = %q, want %q", test.token, got, test.want)
+ }
+ })
+ }
+}
+
+// prefixAlternationPat pulls the prefix alternation out of keyPat's source,
+// e.g. `(?:ghp|gho|ghu|ghs|ghr|github_pat)_` -> `ghp|gho|ghu|ghs|ghr|github_pat`.
+var prefixAlternationPat = regexp.MustCompile(`\(\?:([a-zA-Z0-9_|]+)\)_`)
+
+// keyPatPrefixes derives the token prefixes directly from keyPat's own source
+// rather than from a hand-maintained list. If the two disagree, the regex is
+// the authority — it is what actually decides whether a token is matched at
+// all.
+func keyPatPrefixes(t *testing.T) []string {
+ t.Helper()
+
+ m := prefixAlternationPat.FindStringSubmatch(keyPat.String())
+ if m == nil {
+ t.Fatalf("could not find a prefix alternation group in keyPat: %s\n"+
+ "keyPat's shape changed; update prefixAlternationPat to match, or this "+
+ "drift guard is silently testing nothing.", keyPat.String())
+ }
+
+ var prefixes []string
+ for _, alt := range strings.Split(m[1], "|") {
+ prefixes = append(prefixes, alt+"_")
+ }
+ return prefixes
+}
+
+// TestGithubTokenType_MappingMatchesKeyPatExactly is the drift guard: it
+// derives the prefix set from keyPat itself rather than a hand-maintained
+// list, so a seventh prefix added to keyPat and forgotten in the map cannot
+// slip past silently — the regex and the map cannot diverge without a
+// failure.
+//
+// This matters more than a normal mapping test: an unmapped prefix does not
+// crash, it silently reports "Unknown GitHub token", which puts a responder
+// back in the position this detector change exists to fix — holding a real
+// leaked credential with no idea how to revoke it.
+func TestGithubTokenType_MappingMatchesKeyPatExactly(t *testing.T) {
+ prefixes := keyPatPrefixes(t)
+
+ if len(prefixes) == 0 {
+ t.Fatal("derived zero prefixes from keyPat; the guard is not testing anything")
+ }
+
+ fromKeyPat := make(map[string]bool, len(prefixes))
+ for _, p := range prefixes {
+ fromKeyPat[p] = true
+ if _, ok := tokenTypesByPrefix[p]; !ok {
+ t.Errorf("keyPat matches prefix %q but tokenTypesByPrefix has no entry for it; "+
+ "a leaked token with this prefix would be reported as %q",
+ p, "Unknown GitHub token")
+ }
+ }
+
+ // The reverse direction: a mapping entry for a prefix keyPat cannot match is
+ // dead weight, and usually means a prefix was renamed in one place only.
+ for p := range tokenTypesByPrefix {
+ if !fromKeyPat[p] {
+ t.Errorf("tokenTypesByPrefix has an entry for %q, but keyPat never matches that prefix", p)
+ }
+ }
+}
+
+// assertNoBlankFields checks that v is not a blank string, or — if v is a
+// struct — that none of its exported fields are blank strings. It exists so
+// that if a row's value type grows a second field (tokenTypesByPrefix held a
+// {TokenType, Remediation} struct before remediation was removed, and could
+// again), row validation below catches an unset field automatically instead
+// of depending on someone remembering to add a field-specific check.
+func assertNoBlankFields(t *testing.T, label string, v reflect.Value) {
+ t.Helper()
+
+ switch v.Kind() {
+ case reflect.String:
+ if strings.TrimSpace(v.String()) == "" {
+ t.Errorf("%s: blank value", label)
+ }
+ case reflect.Struct:
+ vt := v.Type()
+ for i := 0; i < v.NumField(); i++ {
+ field := vt.Field(i)
+ if !field.IsExported() {
+ continue
+ }
+ assertNoBlankFields(t, label+"."+field.Name, v.Field(i))
+ }
+ default:
+ t.Fatalf("%s: assertNoBlankFields does not know how to validate kind %s; extend the helper for the new row type", label, v.Kind())
+ }
+}
+
+// TestTokenTypesByPrefix_RowsAreWellFormed checks each row in
+// tokenTypesByPrefix in isolation, independent of keyPat. The tests above
+// confirm the map's keys line up with keyPat; none of them would catch a
+// malformed row added alongside a correct key — e.g. a copy-pasted value
+// that leaves two different prefixes describing the same token type, or a
+// field left blank.
+func TestTokenTypesByPrefix_RowsAreWellFormed(t *testing.T) {
+ const unknown = "Unknown GitHub token"
+
+ seenValues := make(map[string]string, len(tokenTypesByPrefix)) // value -> first prefix that used it
+
+ for prefix, tokenType := range tokenTypesByPrefix {
+ if prefix == "" {
+ t.Error("tokenTypesByPrefix has an empty-string prefix key")
+ }
+ if !strings.HasSuffix(prefix, "_") {
+ t.Errorf("prefix %q does not end in \"_\"; githubTokenType matches by strings.HasPrefix, so a bare prefix can match unrelated tokens", prefix)
+ }
+ assertNoBlankFields(t, prefix, reflect.ValueOf(tokenType))
+ if tokenType == unknown {
+ t.Errorf("prefix %q maps to the reserved fallback value %q; a real row must not collide with it", prefix, unknown)
+ }
+ if other, dup := seenValues[tokenType]; dup {
+ t.Errorf("prefixes %q and %q both map to token type %q; each prefix must describe a distinct credential", prefix, other, tokenType)
+ } else {
+ seenValues[tokenType] = prefix
+ }
+ }
+}
+
+// TestGithubTokenType_EveryKeyPatPrefixResolves closes the loop end to end:
+// build a token for each prefix keyPat actually accepts, confirm keyPat
+// matches it, and confirm githubTokenType resolves it to something other than
+// the unknown fallback. The two tests above check the data structures agree;
+// this one checks the behaviour they produce is correct for every real input.
+func TestGithubTokenType_EveryKeyPatPrefixResolves(t *testing.T) {
+ const unknown = "Unknown GitHub token"
+
+ for _, prefix := range keyPatPrefixes(t) {
+ t.Run(prefix, func(t *testing.T) {
+ token := prefix + strings.Repeat("a", 36)
+
+ if !keyPat.MatchString(token) {
+ t.Fatalf("keyPat does not match %q built from its own prefix alternation; "+
+ "the derived prefix is wrong or keyPat's body changed", token)
+ }
+
+ got := githubTokenType(token)
+ if got == unknown {
+ t.Errorf("githubTokenType(%q) = %q; every prefix keyPat matches must resolve to a real type", token, got)
+ }
+ if got == "" {
+ t.Errorf("githubTokenType(%q) returned an empty type", token)
+ }
+ })
+ }
+}