Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
detectors/github/v2: differentiate token type in ExtraData (#5223)
Shawn Hartsock8ee38640f6e6
diff --git a/pkg/detectors/github/v2/github.go b/pkg/detectors/github/v2/github.goindex bd3581237..e20ef6c28 100644--- a/pkg/detectors/github/v2/github.go+++ b/pkg/detectors/github/v2/github.go@@ -3,6 +3,7 @@ package githubimport ("context""fmt"+ "strings"regexp "github.com/wasilibs/go-re2"@@ -37,8 +38,34 @@ var (// TODO: Oauth2 client_id and client_secret// https://developer.github.com/v3/#oauth2-keysecret++ // tokenTypesByPrefix maps each GitHub token prefix to a human-readable type.+ // Each prefix identifies a materially different credential (PAT, OAuth+ // grant, or GitHub App token) that is revoked/rotated through a different+ // GitHub settings page.+ // https://github.blog/2021-04-05-behind-githubs-new-authentication-token-formats/+ tokenTypesByPrefix = map[string]string{+ "ghp_": "Personal Access Token (classic)",+ "github_pat_": "Personal Access Token (fine-grained)",+ "gho_": "OAuth Access Token",+ "ghu_": "GitHub App User-to-Server Token",+ "ghs_": "GitHub App Server-to-Server (installation) Token",+ "ghr_": "GitHub App Refresh Token",+ })+// githubTokenType returns the human-readable token type for a matched token,+// keyed off its prefix. Falls back to a generic label if no known prefix+// matches (should not happen given keyPat, but keeps this safe).+func githubTokenType(token string) string {+ for prefix, tokenType := range tokenTypesByPrefix {+ if strings.HasPrefix(token, prefix) {+ return tokenType+ }+ }+ return "Unknown GitHub token"+}+// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {@@ -62,6 +89,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/github/","version": fmt.Sprintf("%d", s.Version()),+ "token_type": githubTokenType(token),},SecretParts: map[string]string{"key": token},}diff --git a/pkg/detectors/github/v2/github_integration_test.go b/pkg/detectors/github/v2/github_integration_test.goindex 61861f569..20715b6a7 100644--- a/pkg/detectors/github/v2/github_integration_test.go+++ b/pkg/detectors/github/v2/github_integration_test.go@@ -62,9 +62,10 @@ func TestGitHub_FromChunk(t *testing.T) {"rotation_guide": "https://howtorotate.com/docs/tutorials/github/","scopes": "notifications",// "site_admin": "false", // not present in test verifiedGhp- "url": "https://github.com/truffle-sandbox",- "username": "truffle-sandbox",- "version": "2",+ "token_type": "Personal Access Token (classic)",+ "url": "https://github.com/truffle-sandbox",+ "username": "truffle-sandbox",+ "version": "2",},},},@@ -84,6 +85,7 @@ func TestGitHub_FromChunk(t *testing.T) {Verified: false,ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",+ "token_type": "Personal Access Token (classic)","version": "2",},},@@ -104,6 +106,7 @@ func TestGitHub_FromChunk(t *testing.T) {Verified: false,ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",+ "token_type": "OAuth Access Token","version": "2",},},@@ -124,6 +127,7 @@ func TestGitHub_FromChunk(t *testing.T) {Verified: false,ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",+ "token_type": "GitHub App User-to-Server Token","version": "2",},},@@ -144,6 +148,7 @@ func TestGitHub_FromChunk(t *testing.T) {Verified: false,ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",+ "token_type": "GitHub App Server-to-Server (installation) Token","version": "2",},},@@ -164,6 +169,7 @@ func TestGitHub_FromChunk(t *testing.T) {Verified: false,ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",+ "token_type": "GitHub App Refresh Token","version": "2",},},@@ -184,6 +190,7 @@ func TestGitHub_FromChunk(t *testing.T) {Verified: false,ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/github/",+ "token_type": "Personal Access Token (classic)","version": "2",},},diff --git a/pkg/detectors/github/v2/github_test.go b/pkg/detectors/github/v2/github_test.goindex cc769e82a..263eed832 100644--- a/pkg/detectors/github/v2/github_test.go+++ b/pkg/detectors/github/v2/github_test.go@@ -2,8 +2,12 @@ package githubimport ("context"+ "reflect"+ "strings""testing"+ regexp "github.com/wasilibs/go-re2"+"github.com/google/go-cmp/cmp""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"@@ -86,3 +90,175 @@ func TestGithub_Pattern(t *testing.T) {})}}++func TestGithubTokenType(t *testing.T) {+ tests := []struct {+ prefix string+ token string+ want string+ }{+ {"ghp_", "ghp_" + strings.Repeat("a", 36), "Personal Access Token (classic)"},+ {"github_pat_", "github_pat_" + strings.Repeat("a", 36), "Personal Access Token (fine-grained)"},+ {"gho_", "gho_" + strings.Repeat("a", 36), "OAuth Access Token"},+ {"ghu_", "ghu_" + strings.Repeat("a", 36), "GitHub App User-to-Server Token"},+ {"ghs_", "ghs_" + strings.Repeat("a", 36), "GitHub App Server-to-Server (installation) Token"},+ {"ghr_", "ghr_" + strings.Repeat("a", 36), "GitHub App Refresh Token"},+ {"unknown", "not_a_github_token", "Unknown GitHub token"},+ }++ for _, test := range tests {+ t.Run(test.prefix, func(t *testing.T) {+ got := githubTokenType(test.token)+ if got != test.want {+ t.Errorf("githubTokenType(%q) = %q, want %q", test.token, got, test.want)+ }+ })+ }+}++// prefixAlternationPat pulls the prefix alternation out of keyPat's source,+// e.g. `(?:ghp|gho|ghu|ghs|ghr|github_pat)_` -> `ghp|gho|ghu|ghs|ghr|github_pat`.+var prefixAlternationPat = regexp.MustCompile(`\(\?:([a-zA-Z0-9_|]+)\)_`)++// keyPatPrefixes derives the token prefixes directly from keyPat's own source+// rather than from a hand-maintained list. If the two disagree, the regex is+// the authority — it is what actually decides whether a token is matched at+// all.+func keyPatPrefixes(t *testing.T) []string {+ t.Helper()++ m := prefixAlternationPat.FindStringSubmatch(keyPat.String())+ if m == nil {+ t.Fatalf("could not find a prefix alternation group in keyPat: %s\n"++ "keyPat's shape changed; update prefixAlternationPat to match, or this "++ "drift guard is silently testing nothing.", keyPat.String())+ }++ var prefixes []string+ for _, alt := range strings.Split(m[1], "|") {+ prefixes = append(prefixes, alt+"_")+ }+ return prefixes+}++// TestGithubTokenType_MappingMatchesKeyPatExactly is the drift guard: it+// derives the prefix set from keyPat itself rather than a hand-maintained+// list, so a seventh prefix added to keyPat and forgotten in the map cannot+// slip past silently — the regex and the map cannot diverge without a+// failure.+//+// This matters more than a normal mapping test: an unmapped prefix does not+// crash, it silently reports "Unknown GitHub token", which puts a responder+// back in the position this detector change exists to fix — holding a real+// leaked credential with no idea how to revoke it.+func TestGithubTokenType_MappingMatchesKeyPatExactly(t *testing.T) {+ prefixes := keyPatPrefixes(t)++ if len(prefixes) == 0 {+ t.Fatal("derived zero prefixes from keyPat; the guard is not testing anything")+ }++ fromKeyPat := make(map[string]bool, len(prefixes))+ for _, p := range prefixes {+ fromKeyPat[p] = true+ if _, ok := tokenTypesByPrefix[p]; !ok {+ t.Errorf("keyPat matches prefix %q but tokenTypesByPrefix has no entry for it; "++ "a leaked token with this prefix would be reported as %q",+ p, "Unknown GitHub token")+ }+ }++ // The reverse direction: a mapping entry for a prefix keyPat cannot match is+ // dead weight, and usually means a prefix was renamed in one place only.+ for p := range tokenTypesByPrefix {+ if !fromKeyPat[p] {+ t.Errorf("tokenTypesByPrefix has an entry for %q, but keyPat never matches that prefix", p)+ }+ }+}++// assertNoBlankFields checks that v is not a blank string, or — if v is a+// struct — that none of its exported fields are blank strings. It exists so+// that if a row's value type grows a second field (tokenTypesByPrefix held a+// {TokenType, Remediation} struct before remediation was removed, and could+// again), row validation below catches an unset field automatically instead+// of depending on someone remembering to add a field-specific check.+func assertNoBlankFields(t *testing.T, label string, v reflect.Value) {+ t.Helper()++ switch v.Kind() {+ case reflect.String:+ if strings.TrimSpace(v.String()) == "" {+ t.Errorf("%s: blank value", label)+ }+ case reflect.Struct:+ vt := v.Type()+ for i := 0; i < v.NumField(); i++ {+ field := vt.Field(i)+ if !field.IsExported() {+ continue+ }+ assertNoBlankFields(t, label+"."+field.Name, v.Field(i))+ }+ default:+ t.Fatalf("%s: assertNoBlankFields does not know how to validate kind %s; extend the helper for the new row type", label, v.Kind())+ }+}++// TestTokenTypesByPrefix_RowsAreWellFormed checks each row in+// tokenTypesByPrefix in isolation, independent of keyPat. The tests above+// confirm the map's keys line up with keyPat; none of them would catch a+// malformed row added alongside a correct key — e.g. a copy-pasted value+// that leaves two different prefixes describing the same token type, or a+// field left blank.+func TestTokenTypesByPrefix_RowsAreWellFormed(t *testing.T) {+ const unknown = "Unknown GitHub token"++ seenValues := make(map[string]string, len(tokenTypesByPrefix)) // value -> first prefix that used it++ for prefix, tokenType := range tokenTypesByPrefix {+ if prefix == "" {+ t.Error("tokenTypesByPrefix has an empty-string prefix key")+ }+ if !strings.HasSuffix(prefix, "_") {+ t.Errorf("prefix %q does not end in \"_\"; githubTokenType matches by strings.HasPrefix, so a bare prefix can match unrelated tokens", prefix)+ }+ assertNoBlankFields(t, prefix, reflect.ValueOf(tokenType))+ if tokenType == unknown {+ t.Errorf("prefix %q maps to the reserved fallback value %q; a real row must not collide with it", prefix, unknown)+ }+ if other, dup := seenValues[tokenType]; dup {+ t.Errorf("prefixes %q and %q both map to token type %q; each prefix must describe a distinct credential", prefix, other, tokenType)+ } else {+ seenValues[tokenType] = prefix+ }+ }+}++// TestGithubTokenType_EveryKeyPatPrefixResolves closes the loop end to end:+// build a token for each prefix keyPat actually accepts, confirm keyPat+// matches it, and confirm githubTokenType resolves it to something other than+// the unknown fallback. The two tests above check the data structures agree;+// this one checks the behaviour they produce is correct for every real input.+func TestGithubTokenType_EveryKeyPatPrefixResolves(t *testing.T) {+ const unknown = "Unknown GitHub token"++ for _, prefix := range keyPatPrefixes(t) {+ t.Run(prefix, func(t *testing.T) {+ token := prefix + strings.Repeat("a", 36)++ if !keyPat.MatchString(token) {+ t.Fatalf("keyPat does not match %q built from its own prefix alternation; "++ "the derived prefix is wrong or keyPat's body changed", token)+ }++ got := githubTokenType(token)+ if got == unknown {+ t.Errorf("githubTokenType(%q) = %q; every prefix keyPat matches must resolve to a real type", token, got)+ }+ if got == "" {+ t.Errorf("githubTokenType(%q) returned an empty type", token)+ }+ })+ }+}