Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
pull out verification logic from github detectors (#2554)
āh̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d2ef7a4a49fcb
diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.goindex e235ddcb2..8989bac47 100644--- a/pkg/detectors/github/v1/github_old.go+++ b/pkg/detectors/github/v1/github_old.go@@ -33,7 +33,7 @@ var ()// TODO: Add secret context?? Information about access, ownership etc-type userRes struct {+type UserRes struct {Login string `json:"login"`Type string `json:"type"`SiteAdmin bool `json:"site_admin"`@@ -44,6 +44,11 @@ type userRes struct {LdapDN string `json:"ldap_dn"`}+type HeaderInfo struct {+ Scopes string `json:"X-OAuth-Scopes"`+ Expiry string `json:"github-authentication-token-expiration"`+}+// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {@@ -72,62 +77,24 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1 := detectors.Result{DetectorType: detectorspb.DetectorType_Github,Raw: []byte(token),- }- s1.ExtraData = map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/github/",- "version": fmt.Sprintf("%d", s.Version()),+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/github/",+ "version": fmt.Sprintf("%d", s.Version()),+ },}if verify {client := common.SaneHttpClient()- // https://developer.github.com/v3/users/#get-the-authenticated-user- for _, url := range s.Endpoints(s.DefaultEndpoint()) {- req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)- if err != nil {- continue- }- req.Header.Set("Content-Type", "application/json; charset=utf-8")- req.Header.Set("Authorization", fmt.Sprintf("token %s", token))- res, err := client.Do(req)- if err == nil {- if res.StatusCode >= 200 && res.StatusCode < 300 {- var userResponse userRes- err = json.NewDecoder(res.Body).Decode(&userResponse)- res.Body.Close()- if err == nil {- s1.Verified = true-- if err == nil {- s1.Verified = true- s1.ExtraData["username"] = userResponse.Login- s1.ExtraData["url"] = userResponse.UserURL- s1.ExtraData["account_type"] = userResponse.Type- if userResponse.SiteAdmin {- s1.ExtraData["site_admin"] = "true"- }- if userResponse.Name != "" {- s1.ExtraData["name"] = userResponse.Name- }- if userResponse.Company != "" {- s1.ExtraData["company"] = userResponse.Company- }- if userResponse.LdapDN != "" {- s1.ExtraData["ldap_dn"] = userResponse.LdapDN- }-- // GitHub does not seem to consistently return this header.- scopes := res.Header.Get("X-OAuth-Scopes")- if scopes != "" {- s1.ExtraData["scopes"] = scopes- }- expiry := res.Header.Get("github-authentication-token-expiration")- if expiry != "" {- s1.ExtraData["expires_at"] = expiry- }- }- }- }- }++ isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)+ s1.Verified = isVerified+ s1.SetVerificationError(err, token)++ if userResponse != nil {+ SetUserResponse(userResponse, &s1)+ }+ if headers != nil {+ SetHeaderInfo(headers, &s1)}}@@ -141,6 +108,67 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}+func (s Scanner) VerifyGithub(ctx context.Context, client *http.Client, token string) (bool, *UserRes, *HeaderInfo, error) {+ // https://developer.github.com/v3/users/#get-the-authenticated-user+ var requestErr error+ for _, url := range s.Endpoints(s.DefaultEndpoint()) {+ requestErr = nil++ req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)+ if err != nil {+ continue+ }++ req.Header.Set("Content-Type", "application/json; charset=utf-8")+ req.Header.Set("Authorization", fmt.Sprintf("token %s", token))+ res, err := client.Do(req)+ if err != nil {+ requestErr = err+ }++ if res.StatusCode >= 200 && res.StatusCode < 300 {+ var userResponse UserRes+ err = json.NewDecoder(res.Body).Decode(&userResponse)+ res.Body.Close()+ if err == nil {+ // GitHub does not seem to consistently return this header.+ scopes := res.Header.Get("X-OAuth-Scopes")+ expiry := res.Header.Get("github-authentication-token-expiration")+ return true, &userResponse, &HeaderInfo{Scopes: scopes, Expiry: expiry}, nil+ }+ }+ }+ return false, nil, nil, requestErr+}++func SetUserResponse(userResponse *UserRes, s1 *detectors.Result) {+ s1.ExtraData["username"] = userResponse.Login+ s1.ExtraData["url"] = userResponse.UserURL+ s1.ExtraData["account_type"] = userResponse.Type++ if userResponse.SiteAdmin {+ s1.ExtraData["site_admin"] = "true"+ }+ if userResponse.Name != "" {+ s1.ExtraData["name"] = userResponse.Name+ }+ if userResponse.Company != "" {+ s1.ExtraData["company"] = userResponse.Company+ }+ if userResponse.LdapDN != "" {+ s1.ExtraData["ldap_dn"] = userResponse.LdapDN+ }+}++func SetHeaderInfo(headers *HeaderInfo, s1 *detectors.Result) {+ if headers.Scopes != "" {+ s1.ExtraData["scopes"] = headers.Scopes+ }+ if headers.Expiry != "" {+ s1.ExtraData["expiry"] = headers.Expiry+ }+}+func (s Scanner) Type() detectorspb.DetectorType {return detectorspb.DetectorType_Github}diff --git a/pkg/detectors/github/v1/github_old_test.go b/pkg/detectors/github/v1/github_old_test.goindex b1d3470d0..02ed8aef1 100644--- a/pkg/detectors/github/v1/github_old_test.go+++ b/pkg/detectors/github/v1/github_old_test.go@@ -16,6 +16,8 @@ import ()func TestGitHub_FromChunk(t *testing.T) {+ t.Skip("old tokens no longer valid, and no way to generate new ones")+ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")diff --git a/pkg/detectors/github/v2/github.go b/pkg/detectors/github/v2/github.goindex 5b2d4778d..912c17a20 100644--- a/pkg/detectors/github/v2/github.go+++ b/pkg/detectors/github/v2/github.go@@ -2,9 +2,7 @@ package githubimport ("context"- "encoding/json""fmt"- "net/http"regexp "github.com/wasilibs/go-re2"@@ -12,16 +10,21 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb""github.com/trufflesecurity/trufflehog/v3/pkg/common"+ v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/github/v1")-type Scanner struct{ detectors.EndpointSetter }+type Scanner struct {+ v1.Scanner+}// Ensure the Scanner satisfies the interfaces at compile time.-var _ detectors.Detector = (*Scanner)(nil)-var _ detectors.Versioner = (*Scanner)(nil)-var _ detectors.EndpointCustomizer = (*Scanner)(nil)+var _ detectors.Detector = (*v1.Scanner)(nil)+var _ detectors.Versioner = (*v1.Scanner)(nil)+var _ detectors.EndpointCustomizer = (*v1.Scanner)(nil)-func (Scanner) Version() int { return 2 }+func (s Scanner) Version() int {+ return 2+}func (Scanner) DefaultEndpoint() string { return "https://api.github.com" }var (@@ -36,18 +39,6 @@ var (// https://developer.github.com/v3/#oauth2-keysecret)-// TODO: Add secret context?? Information about access, ownership etc-type userRes struct {- Login string `json:"login"`- Type string `json:"type"`- SiteAdmin bool `json:"site_admin"`- Name string `json:"name"`- Company string `json:"company"`- UserURL string `json:"html_url"`- // Included in GitHub Enterprise Server.- LdapDN string `json:"ldap_dn"`-}-// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {@@ -79,60 +70,22 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif verify {client := common.SaneHttpClient()- // https://developer.github.com/v3/users/#get-the-authenticated-user- for _, url := range s.Endpoints(s.DefaultEndpoint()) {- req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("%s/user", url), nil)- if err != nil {- continue- }-- req.Header.Set("Content-Type", "application/json; charset=utf-8")- req.Header.Set("Authorization", fmt.Sprintf("token %s", token))- res, err := client.Do(req)- if err == nil {- if res.StatusCode >= 200 && res.StatusCode < 300 {- var userResponse userRes- err = json.NewDecoder(res.Body).Decode(&userResponse)- res.Body.Close()- if err == nil {- s1.Verified = true- s1.ExtraData["username"] = userResponse.Login- s1.ExtraData["url"] = userResponse.UserURL- s1.ExtraData["account_type"] = userResponse.Type- if userResponse.SiteAdmin {- s1.ExtraData["site_admin"] = "true"- }- if userResponse.Name != "" {- s1.ExtraData["name"] = userResponse.Name- }- if userResponse.Company != "" {- s1.ExtraData["company"] = userResponse.Company- }- if userResponse.LdapDN != "" {- s1.ExtraData["ldap_dn"] = userResponse.LdapDN- }-- // GitHub does not seem to consistently return this header.- scopes := res.Header.Get("X-OAuth-Scopes")- if scopes != "" {- s1.ExtraData["scopes"] = scopes- }- expiry := res.Header.Get("github-authentication-token-expiration")- if expiry != "" {- s1.ExtraData["expires_at"] = expiry- }- }- }- } else {- s1.SetVerificationError(err, token)- }++ isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)+ s1.Verified = isVerified+ s1.SetVerificationError(err, token)++ if userResponse != nil {+ v1.SetUserResponse(userResponse, &s1)+ }+ if headers != nil {+ v1.SetHeaderInfo(headers, &s1)}}if !s1.Verified && detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {continue}-results = append(results, s1)}