Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Fix NetSuite detector running out of memory on dense input (#5383)
Shahzad Haider2b2c270bede5
diff --git a/pkg/detectors/netsuite/netsuite.go b/pkg/detectors/netsuite/netsuite.goindex 82ce63ef5..41b8a04b6 100644--- a/pkg/detectors/netsuite/netsuite.go+++ b/pkg/detectors/netsuite/netsuite.go@@ -7,7 +7,10 @@ import ("crypto/sha256""encoding/base64""encoding/binary"+ "errors""fmt"+ "io"+ "net""net/http""net/url""strconv"@@ -16,6 +19,7 @@ import (regexp "github.com/wasilibs/go-re2"+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"@@ -41,6 +45,9 @@ var (tokenSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netsuite", "token", "secret"}) + `\b([a-zA-Z0-9]{64})\b`)accountIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netsuite", "account", "id"}) + `\b([a-zA-Z0-9-_]{6,15})\b`)++ // invalidHosts holds account hosts that do not resolve, so later combinations skip the lookup.+ invalidHosts = simple.NewCache[struct{}]())type credentialSet struct {@@ -68,8 +75,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resulttokenSecretMatches := trimUniqueMatches(tokenSecretPat.FindAllStringSubmatch(dataStr, -1))accountIDMatches := trimUniqueMatches(accountIDPat.FindAllStringSubmatch(dataStr, -1))+ // Every combination needs a token key, token secret and account ID.+ if len(tokenKeyMatches) == 0 || len(tokenSecretMatches) == 0 || len(accountIDMatches) == 0 {+ return nil, nil+ }++ // Raw and RawV2 hold only the consumer key and secret, so every combination for a pair is a duplicate of the same+ // finding. Report one result per pair rather than one per combination of all five parts. The token key, token+ // secret and account ID decide whether the pair is reported and whether it verifies.for consumerKey := range consumerKeyMatches {for consumerSecret := range consumerSecretMatches {+ // No combination can use the same value as both consumer key and secret.+ if consumerKey == consumerSecret {+ continue+ }++ var pairResult *detectors.Result+ combinations:for tokenKey := range tokenKeyMatches {for tokenSecret := range tokenSecretMatches {for accountID := range accountIDMatches {@@ -107,10 +129,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1.Verified = isVerifieds1.SetVerificationError(err, consumerKey)}- results = append(results, s1)++ // Keep the verified result, else the first indeterminate one, else the first one.+ if pairResult == nil || s1.Verified || (s1.VerificationError() != nil && pairResult.VerificationError() == nil) {+ pairResult = &s1+ }+ if !verify || s1.Verified {+ break combinations+ }+ if err := ctx.Err(); err != nil {+ // Combinations left untried might have verified.+ if pairResult.VerificationError() == nil {+ pairResult.SetVerificationError(err, consumerKey)+ }+ break combinations+ }}}}++ if pairResult != nil {+ results = append(results, *pairResult)+ }}}@@ -126,9 +166,20 @@ func (s Scanner) Description() string {}func verifyCredentials(ctx context.Context, client *http.Client, cs credentialSet) (bool, error) {+ // The engine waits for FromData to return even after its deadline, so send no request once the context is done.+ if err := ctx.Err(); err != nil {+ return false, err+ }+// for url, filter or replace underscore in accountID if needed and lower case the accountIDurlAccountId := strings.ToLower(strings.ReplaceAll(cs.accountID, "_", "-"))+ // An earlier combination already found that this account's host does not resolve, so skip the lookup and+ // return what the lookup would.+ if invalidHosts.Exists(urlAccountId) {+ return false, nil+ }+baseUrl := "https://" + urlAccountId + ".suitetalk.api.netsuite.com"const path = "/services/rest/record/v1/metadata-catalog/check"@@ -164,12 +215,17 @@ func verifyCredentials(ctx context.Context, client *http.Client, cs credentialSe// Make the requestres, err := client.Do(req)if err != nil {- if strings.Contains(err.Error(), "no such host") {+ var dnsErr *net.DNSError+ if errors.As(err, &dnsErr) && dnsErr.IsNotFound {+ invalidHosts.Set(urlAccountId, struct{}{})return false, nil}return false, err}- defer func() { _ = res.Body.Close() }()+ defer func() {+ _, _ = io.Copy(io.Discard, res.Body)+ _ = res.Body.Close()+ }()switch res.StatusCode {case http.StatusOK:return true, nildiff --git a/pkg/detectors/netsuite/netsuite_integration_test.go b/pkg/detectors/netsuite/netsuite_integration_test.goindex 85d2495d9..82d5f5ceb 100644--- a/pkg/detectors/netsuite/netsuite_integration_test.go+++ b/pkg/detectors/netsuite/netsuite_integration_test.go@@ -78,7 +78,7 @@ func TestNetsuite_FromChunk(t *testing.T) {verify: true,},ShouldHaveVerified: false,- wantCount: 21,+ wantCount: 5,wantErr: false,},{diff --git a/pkg/detectors/netsuite/netsuite_test.go b/pkg/detectors/netsuite/netsuite_test.goindex 62e44d4f0..237dd76fa 100644--- a/pkg/detectors/netsuite/netsuite_test.go+++ b/pkg/detectors/netsuite/netsuite_test.go@@ -3,10 +3,16 @@ package netsuiteimport ("context""fmt"+ "io"+ "net"+ "net/http"+ "strings"+ "sync/atomic""testing""github.com/google/go-cmp/cmp"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")@@ -41,7 +47,7 @@ func TestNetsuite_Pattern(t *testing.T) {{name: "valid pattern - with keyword netsuite",input: fmt.Sprintf(inputFormat, keyword, validAccountID, validConsumerKey, validConsumerSecret, validTokenKey, validTokenSecret),- want: []string{outputPair1, outputPair2, outputPair1, outputPair2},+ want: []string{outputPair1, outputPair2},},{name: "invalid pattern",@@ -92,3 +98,202 @@ func TestNetsuite_Pattern(t *testing.T) {})}}++// denseValue returns the i-th 64-character value of tokenDenseInput.+func denseValue(i int) string {+ return fmt.Sprintf("%064x", i)+}++// tokenDenseInput returns n distinct 64-character values, each of which every key and secret pattern matches, and one+// account ID.+func tokenDenseInput(n int) []byte {+ var input strings.Builder+ for i := 1; i <= n; i++ {+ fmt.Fprintf(&input, "netsuite = %s\n", denseValue(i))+ }+ input.WriteString("account_id = 1234567\n")+ return []byte(input.String())+}++// fakeClient returns a client that counts its requests and answers each with the status respond picks.+func fakeClient(requests *atomic.Int32, respond func(req *http.Request) int) *http.Client {+ return &http.Client{+ Transport: common.FakeTransport{+ CreateResponse: func(req *http.Request) (*http.Response, error) {+ requests.Add(1)+ return &http.Response{+ Request: req,+ StatusCode: respond(req),+ Body: io.NopCloser(strings.NewReader("")),+ }, nil+ },+ },+ }+}++func TestNetsuite_TokenDenseInput(t *testing.T) {+ const values = 16++ ctx, cancel := context.WithCancel(context.Background())+ cancel()++ results, err := Scanner{}.FromData(ctx, false, tokenDenseInput(values))+ if err != nil {+ t.Fatalf("error = %v", err)+ }++ // One result per ordered consumer key and secret pair, not one per combination of all five parts.+ if want := values * (values - 1); len(results) != want {+ t.Fatalf("expected %d results, got %d", want, len(results))+ }+ seen := make(map[string]struct{}, len(results))+ for _, r := range results {+ if _, ok := seen[string(r.RawV2)]; ok {+ t.Fatalf("duplicate result %q", r.RawV2)+ }+ seen[string(r.RawV2)] = struct{}{}+ }+}++func TestNetsuite_Verification(t *testing.T) {+ // With 4 values there are 4*3 consumer key and secret pairs, each completed by 2*1 token key and secret orders.+ tests := []struct {+ name string+ values int+ status int+ cancel bool+ wantVerified bool+ wantErr string+ wantRequests int32+ }{+ {+ name: "rejected pairs try every combination",+ values: 4,+ status: http.StatusUnauthorized,+ wantRequests: 12 * 2,+ },+ {+ name: "verification stops at the first accepted combination",+ values: 4,+ status: http.StatusOK,+ wantVerified: true,+ wantRequests: 12,+ },+ {+ name: "canceled context sends no requests",+ values: 16,+ status: http.StatusOK,+ cancel: true,+ wantErr: context.Canceled.Error(),+ wantRequests: 0,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ var requests atomic.Int32+ client := fakeClient(&requests, func(*http.Request) int { return tt.status })++ ctx, cancel := context.WithCancel(context.Background())+ defer cancel()+ if tt.cancel {+ cancel()+ }++ results, err := Scanner{client: client}.FromData(ctx, true, tokenDenseInput(tt.values))+ if err != nil {+ t.Fatalf("error = %v", err)+ }++ if want := tt.values * (tt.values - 1); len(results) != want {+ t.Fatalf("expected %d results, got %d", want, len(results))+ }+ for _, r := range results {+ if r.Verified != tt.wantVerified {+ t.Fatalf("verified = %v, want %v", r.Verified, tt.wantVerified)+ }+ var gotErr string+ if err := r.VerificationError(); err != nil {+ gotErr = err.Error()+ }+ if gotErr != tt.wantErr {+ t.Fatalf("verification error = %q, want %q", gotErr, tt.wantErr)+ }+ }+ if got := requests.Load(); got != tt.wantRequests {+ t.Errorf("requests = %d, want %d", got, tt.wantRequests)+ }+ })+ }+}++func TestNetsuite_VerificationFindsTheAcceptedCombination(t *testing.T) {+ // Accept only consumer key 1 with token key 4, so each pair has at most one accepted combination among the+ // rejected ones.+ var requests atomic.Int32+ client := fakeClient(&requests, func(req *http.Request) int {+ auth := req.Header.Get("Authorization")+ if strings.Contains(auth, `oauth_consumer_key="`+denseValue(1)+`"`) &&+ strings.Contains(auth, `oauth_token="`+denseValue(4)+`"`) {+ return http.StatusOK+ }+ return http.StatusUnauthorized+ })++ results, err := Scanner{client: client}.FromData(context.Background(), true, tokenDenseInput(4))+ if err != nil {+ t.Fatalf("error = %v", err)+ }++ // Token key 4 completes consumer key 1 with secret 2 or 3, but not with secret 4.+ want := map[string]struct{}{+ denseValue(1) + denseValue(2): {},+ denseValue(1) + denseValue(3): {},+ }+ got := make(map[string]struct{})+ for _, r := range results {+ if r.Verified {+ got[string(r.RawV2)] = struct{}{}+ }+ }+ if diff := cmp.Diff(want, got); diff != "" {+ t.Errorf("verified pairs diff: (-want +got)\n%s", diff)+ }+}++func TestNetsuite_VerificationMissingHost(t *testing.T) {+ invalidHosts.Clear()+ t.Cleanup(invalidHosts.Clear)++ var requests atomic.Int32+ client := &http.Client{+ Transport: common.FakeTransport{+ CreateResponse: func(req *http.Request) (*http.Response, error) {+ requests.Add(1)+ return nil, &net.DNSError{Err: "no such host", Name: req.URL.Hostname(), IsNotFound: true}+ },+ },+ }+ s := Scanner{client: client}++ // Scan twice, as if the data were split across two chunks. A missing host is unverified, as before.+ for range 2 {+ results, err := s.FromData(context.Background(), true, tokenDenseInput(4))+ if err != nil {+ t.Fatalf("error = %v", err)+ }+ if len(results) != 12 {+ t.Fatalf("expected 12 results, got %d", len(results))+ }+ for _, r := range results {+ if r.Verified || r.VerificationError() != nil {+ t.Fatalf("verified = %v, verification error = %v, want unverified", r.Verified, r.VerificationError())+ }+ }+ }++ // The first lookup caches the host, so no later combination or chunk looks it up again.+ if got := requests.Load(); got != 1 {+ t.Errorf("requests = %d, want 1", got)+ }+}