Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
feat(mailgun): update detector (#2679)
Richard Gomez034ca350ab44
diff --git a/pkg/detectors/mailgun/mailgun.go b/pkg/detectors/mailgun/mailgun.goindex c9d5ca359..b9d29ecef 100644--- a/pkg/detectors/mailgun/mailgun.go+++ b/pkg/detectors/mailgun/mailgun.go@@ -2,7 +2,9 @@ package mailgunimport ("context"+ "encoding/json""fmt"+ "io""net/http""strings"@@ -15,16 +17,17 @@ import (type Scanner struct {detectors.DefaultMultiPartCredentialProvider+ client *http.Client}// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ defaultClient = common.SaneHttpClient()tokenPats = map[string]*regexp.Regexp{- "Original MailGun Token": regexp.MustCompile(detectors.PrefixRegex([]string{"mailgun"}) + `\b([a-zA-Z-0-9]{72})\b`),+ "Original MailGun Token": regexp.MustCompile(detectors.PrefixRegex([]string{"mailgun"}) + `\b([a-zA-Z0-9-]{72})\b`),"Key-MailGun Token": regexp.MustCompile(`\b(key-[a-z0-9]{32})\b`),"Hex MailGun Token": regexp.MustCompile(`\b([a-f0-9]{32}-[a-f0-9]{8}-[a-f0-9]{8})\b`),}@@ -33,55 +36,120 @@ var (// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {- return []string{"mailgun"}+ return []string{"mailgun", "key-"}+}++func (s Scanner) getClient() *http.Client {+ if s.client != nil {+ return s.client+ }+ return defaultClient}// FromData will find and optionally verify Mailgun secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)+ uniqueMatches := make(map[string]struct{})for _, tokenPat := range tokenPats {- matches := tokenPat.FindAllStringSubmatch(dataStr, -1)- for _, match := range matches {- if len(match) != 2 {- continue- }- resMatch := strings.TrimSpace(match[1])+ for _, match := range tokenPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueMatches[match[1]] = struct{}{}+ }+ }- s1 := detectors.Result{- DetectorType: detectorspb.DetectorType_Mailgun,- Raw: []byte(resMatch),- }+ for match := range uniqueMatches {+ s1 := detectors.Result{+ DetectorType: s.Type(),+ Raw: []byte(match),+ AnalysisInfo: map[string]string{"key": match},+ }- if verify {- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.mailgun.net/v3/domains", nil)- if err != nil {- continue- }+ if verify {+ client := s.getClient()+ isVerified, extraData, verificationErr := verifyMatch(ctx, client, match)+ s1.Verified = isVerified+ s1.ExtraData = extraData+ s1.SetVerificationError(verificationErr)+ }- // If resMatch has "key" prefix, use it as the username for basic auth.- if strings.HasPrefix(resMatch, "key-") {- req.SetBasicAuth("api", resMatch)- } else {- req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch))- }+ results = append(results, s1)+ }- res, err := client.Do(req)- if err == nil {- defer res.Body.Close()- if res.StatusCode >= 200 && res.StatusCode < 300 {- s1.Verified = true- }- }- s1.AnalysisInfo = map[string]string{"key": resMatch}+ return+}- }+func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {+ // https://documentation.mailgun.com/docs/mailgun/api-reference/openapi-final/tag/Domains/+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.mailgun.net/v3/domains", nil)+ if err != nil {+ return false, nil, err+ }++ if len(token) == 72 {+ // This matches prior logic, but may not be correct.+ req.Header.Add("Authorization", fmt.Sprintf("Basic %s", token))+ } else {+ // https://documentation.mailgun.com/docs/mailgun/api-reference/authentication/+ req.SetBasicAuth("api", token)+ }+ req.Header.Add("Content-Type", "application/json")- results = append(results, s1)+ res, err := client.Do(req)+ if err != nil {+ return false, nil, err+ }+ defer func() {+ _, _ = io.Copy(io.Discard, res.Body)+ _ = res.Body.Close()+ }()++ if res.StatusCode == http.StatusOK {+ var domains domainResponse+ if err := json.NewDecoder(res.Body).Decode(&domains); err != nil {+ return false, nil, fmt.Errorf("error decoding response body: %w", err)+ }++ var extraData map[string]string+ if len(domains.Items) > 0 {+ sb := strings.Builder{}+ for i, item := range domains.Items {+ if i != 0 {+ sb.WriteString(", ")+ }+ sb.WriteString(item.Name)+ sb.WriteString(" (")+ sb.WriteString(item.State)+ sb.WriteString(",")+ sb.WriteString(item.Type)+ if item.IsDisabled {+ sb.WriteString(",disabled")+ }+ sb.WriteString(")")+ }+ extraData = map[string]string{+ "Domains": sb.String(),+ }}++ return true, extraData, nil+ } else if res.StatusCode == http.StatusUnauthorized {+ return false, nil, nil+ } else {+ return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)}+}++type domainResponse struct {+ TotalCount int `json:"total_count"`+ Items []item `json:"items"`+}- return results, nil+type item struct {+ ID string `json:"id"`+ IsDisabled bool `json:"is_disabled"`+ Name string `json:"name"`+ State string `json:"state"`+ Type string `json:"type"`}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/detectors/mailgun/mailgun_test.go b/pkg/detectors/mailgun/mailgun_test.goindex 0b887ce2c..f5fd7fd01 100644--- a/pkg/detectors/mailgun/mailgun_test.go+++ b/pkg/detectors/mailgun/mailgun_test.go@@ -9,13 +9,111 @@ import ("testing""time"+ "github.com/google/go-cmp/cmp""github.com/kylelemons/godebug/pretty"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")+func TestMailgun_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+ tests := []struct {+ name string+ input string+ want []string+ }{+ // TODO: Confirm that this is actually an "original token".+ // It's just a hex token encoded as basic auth.+ {+ name: "original token",+ input: `- request:+ method: get+ uri: https://api.mailgun.net/v3/integration-test.domain.invalid/templates/test.template+ body:+ encoding: US-ASCII+ string: ''+ headers:+ Accept:+ - "*/*"+ User-Agent:+ - rest-client/2.1.0 (darwin21.6.0 x86_64) ruby/2.5.1p57+ Accept-Encoding:+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3+ Host:+ - api.mailgun.net+ Authorization:+ - Basic YXBpOmFjZWM0YzA1YjFmMmZjZWJjZmE4ZGE2NDVkYTEwMjMxLTQxM2UzNzNjLTBhYWQzYzM3`,+ want: []string{"YXBpOmFjZWM0YzA1YjFmMmZjZWJjZmE4ZGE2NDVkYTEwMjMxLTQxM2UzNzNjLTBhYWQzYzM3"},+ },+ {+ name: "key- token",+ input: `public static ClientResponse GetBounce() {+ Client client = new Client();+ client.addFilter(new HTTPBasicAuthFilter("api",+ "key-3ax63njp29jz6fds4gc373sgvjxteol1"));+ WebResource webResource =+ client.resource("https://api.mailgun.net/v2/samples.mailgun.org/" ++ "bounces/[email protected]");+ return webResource.get(ClientResponse.class);+}`,+ want: []string{"key-3ax63njp29jz6fds4gc373sgvjxteol1"},+ },+ {+ name: "hex token",+ input: `curl -X POST https://api.mailgun.net/v3/DOMAIN.TEST/messages -u "api:e915b5cdb9a582685d8f3fb1bea0f20f-07bc7b05-f14816a1"`,+ want: []string{"e915b5cdb9a582685d8f3fb1bea0f20f-07bc7b05-f14816a1"},+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2)+ ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors)+ if len(chunkSpecificDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}+func TestMailgun_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()