Skip to content

Find, verify, and analyze leaked credentials

feat(mailgun): update detector (#2679)

Richard Gomez034ca350ab44
diff --git a/pkg/detectors/mailgun/mailgun.go b/pkg/detectors/mailgun/mailgun.go
index c9d5ca359..b9d29ecef 100644
--- a/pkg/detectors/mailgun/mailgun.go
+++ b/pkg/detectors/mailgun/mailgun.go
@@ -2,7 +2,9 @@ package mailgun
import (
"context"
+ "encoding/json"
"fmt"
+ "io"
"net/http"
"strings"
@@ -15,16 +17,17 @@ import (
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
+ client *http.Client
}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ defaultClient = common.SaneHttpClient()
tokenPats = map[string]*regexp.Regexp{
- "Original MailGun Token": regexp.MustCompile(detectors.PrefixRegex([]string{"mailgun"}) + `\b([a-zA-Z-0-9]{72})\b`),
+ "Original MailGun Token": regexp.MustCompile(detectors.PrefixRegex([]string{"mailgun"}) + `\b([a-zA-Z0-9-]{72})\b`),
"Key-MailGun Token": regexp.MustCompile(`\b(key-[a-z0-9]{32})\b`),
"Hex MailGun Token": regexp.MustCompile(`\b([a-f0-9]{32}-[a-f0-9]{8}-[a-f0-9]{8})\b`),
}
@@ -33,55 +36,120 @@ var (
// Keywords are used for efficiently pre-filtering chunks.
// Use identifiers in the secret preferably, or the provider name.
func (s Scanner) Keywords() []string {
- return []string{"mailgun"}
+ return []string{"mailgun", "key-"}
+}
+
+func (s Scanner) getClient() *http.Client {
+ if s.client != nil {
+ return s.client
+ }
+ return defaultClient
}
// FromData will find and optionally verify Mailgun secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
+ uniqueMatches := make(map[string]struct{})
for _, tokenPat := range tokenPats {
- matches := tokenPat.FindAllStringSubmatch(dataStr, -1)
- for _, match := range matches {
- if len(match) != 2 {
- continue
- }
- resMatch := strings.TrimSpace(match[1])
+ for _, match := range tokenPat.FindAllStringSubmatch(dataStr, -1) {
+ uniqueMatches[match[1]] = struct{}{}
+ }
+ }
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_Mailgun,
- Raw: []byte(resMatch),
- }
+ for match := range uniqueMatches {
+ s1 := detectors.Result{
+ DetectorType: s.Type(),
+ Raw: []byte(match),
+ AnalysisInfo: map[string]string{"key": match},
+ }
- if verify {
- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.mailgun.net/v3/domains", nil)
- if err != nil {
- continue
- }
+ if verify {
+ client := s.getClient()
+ isVerified, extraData, verificationErr := verifyMatch(ctx, client, match)
+ s1.Verified = isVerified
+ s1.ExtraData = extraData
+ s1.SetVerificationError(verificationErr)
+ }
- // If resMatch has "key" prefix, use it as the username for basic auth.
- if strings.HasPrefix(resMatch, "key-") {
- req.SetBasicAuth("api", resMatch)
- } else {
- req.Header.Add("Authorization", fmt.Sprintf("Basic %s", resMatch))
- }
+ results = append(results, s1)
+ }
- res, err := client.Do(req)
- if err == nil {
- defer res.Body.Close()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- s1.Verified = true
- }
- }
- s1.AnalysisInfo = map[string]string{"key": resMatch}
+ return
+}
- }
+func verifyMatch(ctx context.Context, client *http.Client, token string) (bool, map[string]string, error) {
+ // https://documentation.mailgun.com/docs/mailgun/api-reference/openapi-final/tag/Domains/
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.mailgun.net/v3/domains", nil)
+ if err != nil {
+ return false, nil, err
+ }
+
+ if len(token) == 72 {
+ // This matches prior logic, but may not be correct.
+ req.Header.Add("Authorization", fmt.Sprintf("Basic %s", token))
+ } else {
+ // https://documentation.mailgun.com/docs/mailgun/api-reference/authentication/
+ req.SetBasicAuth("api", token)
+ }
+ req.Header.Add("Content-Type", "application/json")
- results = append(results, s1)
+ res, err := client.Do(req)
+ if err != nil {
+ return false, nil, err
+ }
+ defer func() {
+ _, _ = io.Copy(io.Discard, res.Body)
+ _ = res.Body.Close()
+ }()
+
+ if res.StatusCode == http.StatusOK {
+ var domains domainResponse
+ if err := json.NewDecoder(res.Body).Decode(&domains); err != nil {
+ return false, nil, fmt.Errorf("error decoding response body: %w", err)
+ }
+
+ var extraData map[string]string
+ if len(domains.Items) > 0 {
+ sb := strings.Builder{}
+ for i, item := range domains.Items {
+ if i != 0 {
+ sb.WriteString(", ")
+ }
+ sb.WriteString(item.Name)
+ sb.WriteString(" (")
+ sb.WriteString(item.State)
+ sb.WriteString(",")
+ sb.WriteString(item.Type)
+ if item.IsDisabled {
+ sb.WriteString(",disabled")
+ }
+ sb.WriteString(")")
+ }
+ extraData = map[string]string{
+ "Domains": sb.String(),
+ }
}
+
+ return true, extraData, nil
+ } else if res.StatusCode == http.StatusUnauthorized {
+ return false, nil, nil
+ } else {
+ return false, nil, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)
}
+}
+
+type domainResponse struct {
+ TotalCount int `json:"total_count"`
+ Items []item `json:"items"`
+}
- return results, nil
+type item struct {
+ ID string `json:"id"`
+ IsDisabled bool `json:"is_disabled"`
+ Name string `json:"name"`
+ State string `json:"state"`
+ Type string `json:"type"`
}
func (s Scanner) Type() detectorspb.DetectorType {
diff --git a/pkg/detectors/mailgun/mailgun_test.go b/pkg/detectors/mailgun/mailgun_test.go
index 0b887ce2c..f5fd7fd01 100644
--- a/pkg/detectors/mailgun/mailgun_test.go
+++ b/pkg/detectors/mailgun/mailgun_test.go
@@ -9,13 +9,111 @@ import (
"testing"
"time"
+ "github.com/google/go-cmp/cmp"
"github.com/kylelemons/godebug/pretty"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
+func TestMailgun_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+ tests := []struct {
+ name string
+ input string
+ want []string
+ }{
+ // TODO: Confirm that this is actually an "original token".
+ // It's just a hex token encoded as basic auth.
+ {
+ name: "original token",
+ input: `- request:
+ method: get
+ uri: https://api.mailgun.net/v3/integration-test.domain.invalid/templates/test.template
+ body:
+ encoding: US-ASCII
+ string: ''
+ headers:
+ Accept:
+ - "*/*"
+ User-Agent:
+ - rest-client/2.1.0 (darwin21.6.0 x86_64) ruby/2.5.1p57
+ Accept-Encoding:
+ - gzip;q=1.0,deflate;q=0.6,identity;q=0.3
+ Host:
+ - api.mailgun.net
+ Authorization:
+ - Basic YXBpOmFjZWM0YzA1YjFmMmZjZWJjZmE4ZGE2NDVkYTEwMjMxLTQxM2UzNzNjLTBhYWQzYzM3`,
+ want: []string{"YXBpOmFjZWM0YzA1YjFmMmZjZWJjZmE4ZGE2NDVkYTEwMjMxLTQxM2UzNzNjLTBhYWQzYzM3"},
+ },
+ {
+ name: "key- token",
+ input: `public static ClientResponse GetBounce() {
+ Client client = new Client();
+ client.addFilter(new HTTPBasicAuthFilter("api",
+ "key-3ax63njp29jz6fds4gc373sgvjxteol1"));
+ WebResource webResource =
+ client.resource("https://api.mailgun.net/v2/samples.mailgun.org/" +
+ "bounces/[email protected]");
+ return webResource.get(ClientResponse.class);
+}`,
+ want: []string{"key-3ax63njp29jz6fds4gc373sgvjxteol1"},
+ },
+ {
+ name: "hex token",
+ input: `curl -X POST https://api.mailgun.net/v3/DOMAIN.TEST/messages -u "api:e915b5cdb9a582685d8f3fb1bea0f20f-07bc7b05-f14816a1"`,
+ want: []string{"e915b5cdb9a582685d8f3fb1bea0f20f-07bc7b05-f14816a1"},
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ chunkSpecificDetectors := make(map[ahocorasick.DetectorKey]detectors.Detector, 2)
+ ahoCorasickCore.PopulateMatchingDetectors(test.input, chunkSpecificDetectors)
+ if len(chunkSpecificDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
+ })
+ }
+}
+
func TestMailgun_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()