Skip to content

Find, verify, and analyze leaked credentials

pull out verification logic from github detectors (#2554)

āh̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d2ef7a4a49fcb
diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.go
index e235ddcb2..8989bac47 100644
--- a/pkg/detectors/github/v1/github_old.go
+++ b/pkg/detectors/github/v1/github_old.go
@@ -33,7 +33,7 @@ var (
)
// TODO: Add secret context?? Information about access, ownership etc
-type userRes struct {
+type UserRes struct {
Login string `json:"login"`
Type string `json:"type"`
SiteAdmin bool `json:"site_admin"`
@@ -44,6 +44,11 @@ type userRes struct {
LdapDN string `json:"ldap_dn"`
}
+type HeaderInfo struct {
+ Scopes string `json:"X-OAuth-Scopes"`
+ Expiry string `json:"github-authentication-token-expiration"`
+}
+
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
@@ -72,62 +77,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Github,
Raw: []byte(token),
- }
- s1.ExtraData = map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
- "version": fmt.Sprintf("%d", s.Version()),
+ ExtraData: map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/",
+ "version": fmt.Sprintf("%d", s.Version()),
+ },
}
if verify {
client := common.SaneHttpClient()
- // https://developer.github.com/v3/users/#get-the-authenticated-user
- for _, url := range s.Endpoints(s.DefaultEndpoint()) {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
- if err != nil {
- continue
- }
- req.Header.Set("Content-Type", "application/json; charset=utf-8")
- req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
- res, err := client.Do(req)
- if err == nil {
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- var userResponse userRes
- err = json.NewDecoder(res.Body).Decode(&userResponse)
- res.Body.Close()
- if err == nil {
- s1.Verified = true
-
- if err == nil {
- s1.Verified = true
- s1.ExtraData["username"] = userResponse.Login
- s1.ExtraData["url"] = userResponse.UserURL
- s1.ExtraData["account_type"] = userResponse.Type
- if userResponse.SiteAdmin {
- s1.ExtraData["site_admin"] = "true"
- }
- if userResponse.Name != "" {
- s1.ExtraData["name"] = userResponse.Name
- }
- if userResponse.Company != "" {
- s1.ExtraData["company"] = userResponse.Company
- }
- if userResponse.LdapDN != "" {
- s1.ExtraData["ldap_dn"] = userResponse.LdapDN
- }
-
- // GitHub does not seem to consistently return this header.
- scopes := res.Header.Get("X-OAuth-Scopes")
- if scopes != "" {
- s1.ExtraData["scopes"] = scopes
- }
- expiry := res.Header.Get("github-authentication-token-expiration")
- if expiry != "" {
- s1.ExtraData["expires_at"] = expiry
- }
- }
- }
- }
- }
+
+ isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)
+ s1.Verified = isVerified
+ s1.SetVerificationError(err, token)
+
+ if userResponse != nil {
+ SetUserResponse(userResponse, &s1)
+ }
+ if headers != nil {
+ SetHeaderInfo(headers, &s1)
}
}
@@ -141,6 +108,67 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
+func (s Scanner) VerifyGithub(ctx context.Context, client *http.Client, token string) (bool, *UserRes, *HeaderInfo, error) {
+ // https://developer.github.com/v3/users/#get-the-authenticated-user
+ var requestErr error
+ for _, url := range s.Endpoints(s.DefaultEndpoint()) {
+ requestErr = nil
+
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
+ if err != nil {
+ continue
+ }
+
+ req.Header.Set("Content-Type", "application/json; charset=utf-8")
+ req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
+ res, err := client.Do(req)
+ if err != nil {
+ requestErr = err
+ }
+
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ var userResponse UserRes
+ err = json.NewDecoder(res.Body).Decode(&userResponse)
+ res.Body.Close()
+ if err == nil {
+ // GitHub does not seem to consistently return this header.
+ scopes := res.Header.Get("X-OAuth-Scopes")
+ expiry := res.Header.Get("github-authentication-token-expiration")
+ return true, &userResponse, &HeaderInfo{Scopes: scopes, Expiry: expiry}, nil
+ }
+ }
+ }
+ return false, nil, nil, requestErr
+}
+
+func SetUserResponse(userResponse *UserRes, s1 *detectors.Result) {
+ s1.ExtraData["username"] = userResponse.Login
+ s1.ExtraData["url"] = userResponse.UserURL
+ s1.ExtraData["account_type"] = userResponse.Type
+
+ if userResponse.SiteAdmin {
+ s1.ExtraData["site_admin"] = "true"
+ }
+ if userResponse.Name != "" {
+ s1.ExtraData["name"] = userResponse.Name
+ }
+ if userResponse.Company != "" {
+ s1.ExtraData["company"] = userResponse.Company
+ }
+ if userResponse.LdapDN != "" {
+ s1.ExtraData["ldap_dn"] = userResponse.LdapDN
+ }
+}
+
+func SetHeaderInfo(headers *HeaderInfo, s1 *detectors.Result) {
+ if headers.Scopes != "" {
+ s1.ExtraData["scopes"] = headers.Scopes
+ }
+ if headers.Expiry != "" {
+ s1.ExtraData["expiry"] = headers.Expiry
+ }
+}
+
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Github
}
diff --git a/pkg/detectors/github/v1/github_old_test.go b/pkg/detectors/github/v1/github_old_test.go
index b1d3470d0..02ed8aef1 100644
--- a/pkg/detectors/github/v1/github_old_test.go
+++ b/pkg/detectors/github/v1/github_old_test.go
@@ -16,6 +16,8 @@ import (
)
func TestGitHub_FromChunk(t *testing.T) {
+ t.Skip("old tokens no longer valid, and no way to generate new ones")
+
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")
diff --git a/pkg/detectors/github/v2/github.go b/pkg/detectors/github/v2/github.go
index 5b2d4778d..912c17a20 100644
--- a/pkg/detectors/github/v2/github.go
+++ b/pkg/detectors/github/v2/github.go
@@ -2,9 +2,7 @@ package github
import (
"context"
- "encoding/json"
"fmt"
- "net/http"
regexp "github.com/wasilibs/go-re2"
@@ -12,16 +10,21 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github/v1"
)
-type Scanner struct{ detectors.EndpointSetter }
+type Scanner struct {
+ v1.Scanner
+}
// Ensure the Scanner satisfies the interfaces at compile time.
-var _ detectors.Detector = (*Scanner)(nil)
-var _ detectors.Versioner = (*Scanner)(nil)
-var _ detectors.EndpointCustomizer = (*Scanner)(nil)
+var _ detectors.Detector = (*v1.Scanner)(nil)
+var _ detectors.Versioner = (*v1.Scanner)(nil)
+var _ detectors.EndpointCustomizer = (*v1.Scanner)(nil)
-func (Scanner) Version() int { return 2 }
+func (s Scanner) Version() int {
+ return 2
+}
func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }
var (
@@ -36,18 +39,6 @@ var (
// https://developer.github.com/v3/#oauth2-keysecret
)
-// TODO: Add secret context?? Information about access, ownership etc
-type userRes struct {
- Login string `json:"login"`
- Type string `json:"type"`
- SiteAdmin bool `json:"site_admin"`
- Name string `json:"name"`
- Company string `json:"company"`
- UserURL string `json:"html_url"`
- // Included in GitHub Enterprise Server.
- LdapDN string `json:"ldap_dn"`
-}
-
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
@@ -79,60 +70,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
client := common.SaneHttpClient()
- // https://developer.github.com/v3/users/#get-the-authenticated-user
- for _, url := range s.Endpoints(s.DefaultEndpoint()) {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)
- if err != nil {
- continue
- }
-
- req.Header.Set("Content-Type", "application/json; charset=utf-8")
- req.Header.Set("Authorization", fmt.Sprintf("token %s", token))
- res, err := client.Do(req)
- if err == nil {
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- var userResponse userRes
- err = json.NewDecoder(res.Body).Decode(&userResponse)
- res.Body.Close()
- if err == nil {
- s1.Verified = true
- s1.ExtraData["username"] = userResponse.Login
- s1.ExtraData["url"] = userResponse.UserURL
- s1.ExtraData["account_type"] = userResponse.Type
- if userResponse.SiteAdmin {
- s1.ExtraData["site_admin"] = "true"
- }
- if userResponse.Name != "" {
- s1.ExtraData["name"] = userResponse.Name
- }
- if userResponse.Company != "" {
- s1.ExtraData["company"] = userResponse.Company
- }
- if userResponse.LdapDN != "" {
- s1.ExtraData["ldap_dn"] = userResponse.LdapDN
- }
-
- // GitHub does not seem to consistently return this header.
- scopes := res.Header.Get("X-OAuth-Scopes")
- if scopes != "" {
- s1.ExtraData["scopes"] = scopes
- }
- expiry := res.Header.Get("github-authentication-token-expiration")
- if expiry != "" {
- s1.ExtraData["expires_at"] = expiry
- }
- }
- }
- } else {
- s1.SetVerificationError(err, token)
- }
+
+ isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)
+ s1.Verified = isVerified
+ s1.SetVerificationError(err, token)
+
+ if userResponse != nil {
+ v1.SetUserResponse(userResponse, &s1)
+ }
+ if headers != nil {
+ v1.SetHeaderInfo(headers, &s1)
}
}
if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
continue
}
-
results = append(results, s1)
}