Skip to content

Find, verify, and analyze leaked credentials

Fix NetSuite detector running out of memory on dense input (#5383)

Shahzad Haider2b2c270bede5
diff --git a/pkg/detectors/netsuite/netsuite.go b/pkg/detectors/netsuite/netsuite.go
index 82ce63ef5..41b8a04b6 100644
--- a/pkg/detectors/netsuite/netsuite.go
+++ b/pkg/detectors/netsuite/netsuite.go
@@ -7,7 +7,10 @@ import (
"crypto/sha256"
"encoding/base64"
"encoding/binary"
+ "errors"
"fmt"
+ "io"
+ "net"
"net/http"
"net/url"
"strconv"
@@ -16,6 +19,7 @@ import (
regexp "github.com/wasilibs/go-re2"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/cache/simple"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detector_typepb"
@@ -41,6 +45,9 @@ var (
tokenSecretPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netsuite", "token", "secret"}) + `\b([a-zA-Z0-9]{64})\b`)
accountIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"netsuite", "account", "id"}) + `\b([a-zA-Z0-9-_]{6,15})\b`)
+
+ // invalidHosts holds account hosts that do not resolve, so later combinations skip the lookup.
+ invalidHosts = simple.NewCache[struct{}]()
)
type credentialSet struct {
@@ -68,8 +75,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
tokenSecretMatches := trimUniqueMatches(tokenSecretPat.FindAllStringSubmatch(dataStr, -1))
accountIDMatches := trimUniqueMatches(accountIDPat.FindAllStringSubmatch(dataStr, -1))
+ // Every combination needs a token key, token secret and account ID.
+ if len(tokenKeyMatches) == 0 || len(tokenSecretMatches) == 0 || len(accountIDMatches) == 0 {
+ return nil, nil
+ }
+
+ // Raw and RawV2 hold only the consumer key and secret, so every combination for a pair is a duplicate of the same
+ // finding. Report one result per pair rather than one per combination of all five parts. The token key, token
+ // secret and account ID decide whether the pair is reported and whether it verifies.
for consumerKey := range consumerKeyMatches {
for consumerSecret := range consumerSecretMatches {
+ // No combination can use the same value as both consumer key and secret.
+ if consumerKey == consumerSecret {
+ continue
+ }
+
+ var pairResult *detectors.Result
+ combinations:
for tokenKey := range tokenKeyMatches {
for tokenSecret := range tokenSecretMatches {
for accountID := range accountIDMatches {
@@ -107,10 +129,28 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1.Verified = isVerified
s1.SetVerificationError(err, consumerKey)
}
- results = append(results, s1)
+
+ // Keep the verified result, else the first indeterminate one, else the first one.
+ if pairResult == nil || s1.Verified || (s1.VerificationError() != nil && pairResult.VerificationError() == nil) {
+ pairResult = &s1
+ }
+ if !verify || s1.Verified {
+ break combinations
+ }
+ if err := ctx.Err(); err != nil {
+ // Combinations left untried might have verified.
+ if pairResult.VerificationError() == nil {
+ pairResult.SetVerificationError(err, consumerKey)
+ }
+ break combinations
+ }
}
}
}
+
+ if pairResult != nil {
+ results = append(results, *pairResult)
+ }
}
}
@@ -126,9 +166,20 @@ func (s Scanner) Description() string {
}
func verifyCredentials(ctx context.Context, client *http.Client, cs credentialSet) (bool, error) {
+ // The engine waits for FromData to return even after its deadline, so send no request once the context is done.
+ if err := ctx.Err(); err != nil {
+ return false, err
+ }
+
// for url, filter or replace underscore in accountID if needed and lower case the accountID
urlAccountId := strings.ToLower(strings.ReplaceAll(cs.accountID, "_", "-"))
+ // An earlier combination already found that this account's host does not resolve, so skip the lookup and
+ // return what the lookup would.
+ if invalidHosts.Exists(urlAccountId) {
+ return false, nil
+ }
+
baseUrl := "https://" + urlAccountId + ".suitetalk.api.netsuite.com"
const path = "/services/rest/record/v1/metadata-catalog/check"
@@ -164,12 +215,17 @@ func verifyCredentials(ctx context.Context, client *http.Client, cs credentialSe
// Make the request
res, err := client.Do(req)
if err != nil {
- if strings.Contains(err.Error(), "no such host") {
+ var dnsErr *net.DNSError
+ if errors.As(err, &dnsErr) && dnsErr.IsNotFound {
+ invalidHosts.Set(urlAccountId, struct{}{})
return false, nil
}
return false, err
}
- defer func() { _ = res.Body.Close() }()
+ defer func() {
+ _, _ = io.Copy(io.Discard, res.Body)
+ _ = res.Body.Close()
+ }()
switch res.StatusCode {
case http.StatusOK:
return true, nil
diff --git a/pkg/detectors/netsuite/netsuite_integration_test.go b/pkg/detectors/netsuite/netsuite_integration_test.go
index 85d2495d9..82d5f5ceb 100644
--- a/pkg/detectors/netsuite/netsuite_integration_test.go
+++ b/pkg/detectors/netsuite/netsuite_integration_test.go
@@ -78,7 +78,7 @@ func TestNetsuite_FromChunk(t *testing.T) {
verify: true,
},
ShouldHaveVerified: false,
- wantCount: 21,
+ wantCount: 5,
wantErr: false,
},
{
diff --git a/pkg/detectors/netsuite/netsuite_test.go b/pkg/detectors/netsuite/netsuite_test.go
index 62e44d4f0..237dd76fa 100644
--- a/pkg/detectors/netsuite/netsuite_test.go
+++ b/pkg/detectors/netsuite/netsuite_test.go
@@ -3,10 +3,16 @@ package netsuite
import (
"context"
"fmt"
+ "io"
+ "net"
+ "net/http"
+ "strings"
+ "sync/atomic"
"testing"
"github.com/google/go-cmp/cmp"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
@@ -41,7 +47,7 @@ func TestNetsuite_Pattern(t *testing.T) {
{
name: "valid pattern - with keyword netsuite",
input: fmt.Sprintf(inputFormat, keyword, validAccountID, validConsumerKey, validConsumerSecret, validTokenKey, validTokenSecret),
- want: []string{outputPair1, outputPair2, outputPair1, outputPair2},
+ want: []string{outputPair1, outputPair2},
},
{
name: "invalid pattern",
@@ -92,3 +98,202 @@ func TestNetsuite_Pattern(t *testing.T) {
})
}
}
+
+// denseValue returns the i-th 64-character value of tokenDenseInput.
+func denseValue(i int) string {
+ return fmt.Sprintf("%064x", i)
+}
+
+// tokenDenseInput returns n distinct 64-character values, each of which every key and secret pattern matches, and one
+// account ID.
+func tokenDenseInput(n int) []byte {
+ var input strings.Builder
+ for i := 1; i <= n; i++ {
+ fmt.Fprintf(&input, "netsuite = %s\n", denseValue(i))
+ }
+ input.WriteString("account_id = 1234567\n")
+ return []byte(input.String())
+}
+
+// fakeClient returns a client that counts its requests and answers each with the status respond picks.
+func fakeClient(requests *atomic.Int32, respond func(req *http.Request) int) *http.Client {
+ return &http.Client{
+ Transport: common.FakeTransport{
+ CreateResponse: func(req *http.Request) (*http.Response, error) {
+ requests.Add(1)
+ return &http.Response{
+ Request: req,
+ StatusCode: respond(req),
+ Body: io.NopCloser(strings.NewReader("")),
+ }, nil
+ },
+ },
+ }
+}
+
+func TestNetsuite_TokenDenseInput(t *testing.T) {
+ const values = 16
+
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+
+ results, err := Scanner{}.FromData(ctx, false, tokenDenseInput(values))
+ if err != nil {
+ t.Fatalf("error = %v", err)
+ }
+
+ // One result per ordered consumer key and secret pair, not one per combination of all five parts.
+ if want := values * (values - 1); len(results) != want {
+ t.Fatalf("expected %d results, got %d", want, len(results))
+ }
+ seen := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if _, ok := seen[string(r.RawV2)]; ok {
+ t.Fatalf("duplicate result %q", r.RawV2)
+ }
+ seen[string(r.RawV2)] = struct{}{}
+ }
+}
+
+func TestNetsuite_Verification(t *testing.T) {
+ // With 4 values there are 4*3 consumer key and secret pairs, each completed by 2*1 token key and secret orders.
+ tests := []struct {
+ name string
+ values int
+ status int
+ cancel bool
+ wantVerified bool
+ wantErr string
+ wantRequests int32
+ }{
+ {
+ name: "rejected pairs try every combination",
+ values: 4,
+ status: http.StatusUnauthorized,
+ wantRequests: 12 * 2,
+ },
+ {
+ name: "verification stops at the first accepted combination",
+ values: 4,
+ status: http.StatusOK,
+ wantVerified: true,
+ wantRequests: 12,
+ },
+ {
+ name: "canceled context sends no requests",
+ values: 16,
+ status: http.StatusOK,
+ cancel: true,
+ wantErr: context.Canceled.Error(),
+ wantRequests: 0,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ var requests atomic.Int32
+ client := fakeClient(&requests, func(*http.Request) int { return tt.status })
+
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ if tt.cancel {
+ cancel()
+ }
+
+ results, err := Scanner{client: client}.FromData(ctx, true, tokenDenseInput(tt.values))
+ if err != nil {
+ t.Fatalf("error = %v", err)
+ }
+
+ if want := tt.values * (tt.values - 1); len(results) != want {
+ t.Fatalf("expected %d results, got %d", want, len(results))
+ }
+ for _, r := range results {
+ if r.Verified != tt.wantVerified {
+ t.Fatalf("verified = %v, want %v", r.Verified, tt.wantVerified)
+ }
+ var gotErr string
+ if err := r.VerificationError(); err != nil {
+ gotErr = err.Error()
+ }
+ if gotErr != tt.wantErr {
+ t.Fatalf("verification error = %q, want %q", gotErr, tt.wantErr)
+ }
+ }
+ if got := requests.Load(); got != tt.wantRequests {
+ t.Errorf("requests = %d, want %d", got, tt.wantRequests)
+ }
+ })
+ }
+}
+
+func TestNetsuite_VerificationFindsTheAcceptedCombination(t *testing.T) {
+ // Accept only consumer key 1 with token key 4, so each pair has at most one accepted combination among the
+ // rejected ones.
+ var requests atomic.Int32
+ client := fakeClient(&requests, func(req *http.Request) int {
+ auth := req.Header.Get("Authorization")
+ if strings.Contains(auth, `oauth_consumer_key="`+denseValue(1)+`"`) &&
+ strings.Contains(auth, `oauth_token="`+denseValue(4)+`"`) {
+ return http.StatusOK
+ }
+ return http.StatusUnauthorized
+ })
+
+ results, err := Scanner{client: client}.FromData(context.Background(), true, tokenDenseInput(4))
+ if err != nil {
+ t.Fatalf("error = %v", err)
+ }
+
+ // Token key 4 completes consumer key 1 with secret 2 or 3, but not with secret 4.
+ want := map[string]struct{}{
+ denseValue(1) + denseValue(2): {},
+ denseValue(1) + denseValue(3): {},
+ }
+ got := make(map[string]struct{})
+ for _, r := range results {
+ if r.Verified {
+ got[string(r.RawV2)] = struct{}{}
+ }
+ }
+ if diff := cmp.Diff(want, got); diff != "" {
+ t.Errorf("verified pairs diff: (-want +got)\n%s", diff)
+ }
+}
+
+func TestNetsuite_VerificationMissingHost(t *testing.T) {
+ invalidHosts.Clear()
+ t.Cleanup(invalidHosts.Clear)
+
+ var requests atomic.Int32
+ client := &http.Client{
+ Transport: common.FakeTransport{
+ CreateResponse: func(req *http.Request) (*http.Response, error) {
+ requests.Add(1)
+ return nil, &net.DNSError{Err: "no such host", Name: req.URL.Hostname(), IsNotFound: true}
+ },
+ },
+ }
+ s := Scanner{client: client}
+
+ // Scan twice, as if the data were split across two chunks. A missing host is unverified, as before.
+ for range 2 {
+ results, err := s.FromData(context.Background(), true, tokenDenseInput(4))
+ if err != nil {
+ t.Fatalf("error = %v", err)
+ }
+ if len(results) != 12 {
+ t.Fatalf("expected 12 results, got %d", len(results))
+ }
+ for _, r := range results {
+ if r.Verified || r.VerificationError() != nil {
+ t.Fatalf("verified = %v, verification error = %v, want unverified", r.Verified, r.VerificationError())
+ }
+ }
+ }
+
+ // The first lookup caches the host, so no later combination or chunk looks it up again.
+ if got := requests.Load(); got != 1 {
+ t.Errorf("requests = %d, want 1", got)
+ }
+}