Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Fix Sumo Logic detector ignoring configured verifier endpoints (#5395)
Dave Hubbardab5ee072e1b7
diff --git a/pkg/detectors/sumologickey/sumologickey.go b/pkg/detectors/sumologickey/sumologickey.goindex 31eadc6b2..2cf01fab5 100644--- a/pkg/detectors/sumologickey/sumologickey.go+++ b/pkg/detectors/sumologickey/sumologickey.go@@ -5,6 +5,7 @@ import ("fmt""io""net/http"+ "net/url""strings""github.com/trufflesecurity/trufflehog/v3/pkg/common"@@ -44,8 +45,10 @@ func (s Scanner) Keywords() []string {return []string{"sumo", "accessId", "accessKey"}}-// Default US API endpoint.-func (Scanner) CloudEndpoint() string { return "api.sumologic.com" }+// CloudEndpoint is the US API endpoint. Endpoints are full URLs so that+// configured verifier endpoints, which may carry a path prefix, are used as+// given.+func (Scanner) CloudEndpoint() string { return "https://api.sumologic.com" }// FromData will find and optionally verify SumoLogicKey secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {@@ -59,50 +62,70 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultfor _, match := range keyPat.FindAllStringSubmatch(dataStr, -1) {keyMatches[match[1]] = struct{}{}}- endpointMatches := make(map[string]struct{})+ hostMatches := make(map[string]struct{})for _, match := range urlPat.FindAllStringSubmatch(dataStr, -1) {- endpointMatches[match[0]] = struct{}{}+ hostMatches[match[0]] = struct{}{}}- if len(endpointMatches) == 0 {- endpointMatches[s.CloudEndpoint()] = struct{}{}+ foundEndpoints := make([]string, 0, len(hostMatches))+ for host := range hostMatches {+ foundEndpoints = append(foundEndpoints, "https://"+host)}+ // RawV2 identifies the secret, so it is built only from what the data+ // says and never from the verification outcome. Otherwise the same key+ // gets a different identity when it flips between verified and+ // unverified (for example when it is revoked), and consumers that dedupe+ // on RawV2 record it as a new secret. The access ID and regional host are+ // included only when the data names exactly one of each. The host is kept+ // bare (no scheme) because existing identities were recorded that way.+ rawV2Id := soleKey(idMatches)+ rawV2URL := soleKey(hostMatches)+for accessKey := range keyMatches {var (- r *detectors.Result- accessId string- apiEndpoint string+ verified bool+ verifiedId string+ verifiedEndpoint string+ lastErr error)- for id := range idMatches {- accessId = id-- for e := range endpointMatches {- apiEndpoint = e+ if verify {+ client := s.client+ if client == nil {+ client = defaultClient+ }- if verify {- client := s.client- if client == nil {- client = defaultClient+ // Endpoints applies the engine's endpoint configuration:+ // configured verifier endpoints, the US cloud endpoint, and+ // regional hosts found in the data, each only when enabled.+ //+ // Only non-nil errors overwrite lastErr so that a clean 401 from+ // the wrong region or access ID doesn't erase a failure (500,+ // timeout) from a pair that might be authoritative.+ endpoints := s.Endpoints(foundEndpoints...)+ verification:+ for id := range idMatches {+ for _, baseURL := range endpoints {+ isVerified, vErr := verifyMatch(ctx, client, baseURL, id, accessKey)+ if vErr != nil {+ lastErr = vErr}-- isVerified, verificationErr := verifyMatch(ctx, client, apiEndpoint, accessId, accessKey)if isVerified {- r = createResult(accessId, accessKey, apiEndpoint, isVerified, verificationErr)+ verified, verifiedId, verifiedEndpoint, lastErr = true, id, baseURL, nil+ break verification}}}}- if r == nil {- // Only include the accessId if we're confident which one it is.- if len(idMatches) != 1 {- accessId = ""- }- if len(endpointMatches) != 1 || apiEndpoint == s.CloudEndpoint() {- apiEndpoint = ""- }- r = createResult(accessId, accessKey, apiEndpoint, false, nil)+ r := createResult(rawV2Id, accessKey, rawV2URL, verified, lastErr)+ // Which access ID and endpoint a key verified with are reported here+ // rather than in RawV2, so they don't affect the secret's identity.+ // RawV2 omits the access ID when the data names several, and keys+ // are managed by access ID, so triage needs it from here.+ if verified {+ r.ExtraData["access_id"] = verifiedId+ r.ExtraData["endpoint"] = verifiedEndpoint}results = append(results, *r)}@@ -110,8 +133,23 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func verifyMatch(ctx context.Context, client *http.Client, endpoint string, id string, key string) (bool, error) {- req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s/api/v1/users", endpoint), nil)+// soleKey returns the only key in m, or "" when m has zero or several keys.+func soleKey(m map[string]struct{}) string {+ if len(m) != 1 {+ return ""+ }+ for k := range m {+ return k+ }+ return ""+}++func verifyMatch(ctx context.Context, client *http.Client, baseURL string, id string, key string) (bool, error) {+ endpoint, err := url.JoinPath(baseURL, "/api/v1/users")+ if err != nil {+ return false, err+ }+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint, nil)if err != nil {return false, err}diff --git a/pkg/detectors/sumologickey/sumologickey_integration_test.go b/pkg/detectors/sumologickey/sumologickey_integration_test.goindex b4a3c9aca..b37cec233 100644--- a/pkg/detectors/sumologickey/sumologickey_integration_test.go+++ b/pkg/detectors/sumologickey/sumologickey_integration_test.go@@ -86,6 +86,9 @@ func TestSumoLogicKey_FromChunk(t *testing.T) {for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {s := Scanner{}+ s.UseCloudEndpoint(true)+ s.SetCloudEndpoint(s.CloudEndpoint())+ s.UseFoundEndpoints(true)got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)if (err != nil) != tt.wantErr {t.Errorf("SumoLogicKey.FromData() error = %v, wantErr %v", err, tt.wantErr)diff --git a/pkg/detectors/sumologickey/sumologickey_test.go b/pkg/detectors/sumologickey/sumologickey_test.goindex 2e0c8f1d8..ce7a016c3 100644--- a/pkg/detectors/sumologickey/sumologickey_test.go+++ b/pkg/detectors/sumologickey/sumologickey_test.go@@ -2,6 +2,8 @@ package sumologickeyimport ("context"+ "net/http"+ "net/http/httptest""testing""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"@@ -10,6 +12,12 @@ import ("github.com/google/go-cmp/cmp")+// verifyInput names one access ID and key and no regional URL, so the only+// endpoints verification can reach are the ones a test configures.+const verifyInput = `sumologic:+ accessId: suDkVYKjXZAwsz+ accessKey: Khk3i2ugMxMgkb8bIA2auj4I8juZ3HiimDNssjzYdGqfizPZcxHK70a0LckgRSCL`+func TestSumoLogicKey_Pattern(t *testing.T) {d := Scanner{}ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})@@ -92,3 +100,129 @@ sumoKey2 = 'Khk3i2ugMxMgkb8bIA2auj4I8juZ3HiimDNssjzYdGqfizPZcxHK21a0LckgRSCL'`,})}}++// Exercise verification against a mock server configured as a verifier+// endpoint, so a request only arrives if FromData verifies through+// Endpoints(). A verified result keeps the same RawV2 an unverified one+// would have, and reports the endpoint in ExtraData instead.+func TestSumoLogicKey_Verification(t *testing.T) {+ tests := []struct {+ name string+ statusCode int+ wantVerified bool+ wantErr bool+ }{+ {"200 - valid key", http.StatusOK, true, false},+ {"401 - invalid key", http.StatusUnauthorized, false, false},+ {"500 - server error", http.StatusInternalServerError, false, true},+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(tt.statusCode)+ }))+ defer ts.Close()++ s := Scanner{}+ _ = s.SetConfiguredEndpoints(ts.URL)++ results, err := s.FromData(context.Background(), true, []byte(verifyInput))+ if err != nil {+ t.Fatalf("FromData error: %v", err)+ }+ if len(results) != 1 {+ t.Fatalf("expected 1 result, got %d", len(results))+ }++ r := results[0]+ if r.Verified != tt.wantVerified {+ t.Errorf("Verified = %v, want %v", r.Verified, tt.wantVerified)+ }+ if tt.wantErr && r.VerificationError() == nil {+ t.Error("expected verification error, got nil")+ }+ if !tt.wantErr && r.VerificationError() != nil {+ t.Errorf("unexpected verification error: %v", r.VerificationError())+ }+ wantRawV2 := `{"accessId":"suDkVYKjXZAwsz","accessKey":"Khk3i2ugMxMgkb8bIA2auj4I8juZ3HiimDNssjzYdGqfizPZcxHK70a0LckgRSCL"}`+ if string(r.RawV2) != wantRawV2 {+ t.Errorf("RawV2 = %s, want %s", r.RawV2, wantRawV2)+ }+ if tt.wantVerified && r.ExtraData["endpoint"] != ts.URL {+ t.Errorf("ExtraData[endpoint] = %q, want %q", r.ExtraData["endpoint"], ts.URL)+ }+ if tt.wantVerified && r.ExtraData["access_id"] != "suDkVYKjXZAwsz" {+ t.Errorf("ExtraData[access_id] = %q, want %q", r.ExtraData["access_id"], "suDkVYKjXZAwsz")+ }+ })+ }+}++// Verify that a successful verification on a later endpoint clears any+// error from an earlier failed attempt (e.g. first endpoint returns 500,+// second returns 200).+func TestSumoLogicKey_Verification_StaleErrorCleared(t *testing.T) {+ ts500 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusInternalServerError)+ }))+ defer ts500.Close()++ ts200 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusOK)+ }))+ defer ts200.Close()++ s := Scanner{}+ _ = s.SetConfiguredEndpoints(ts500.URL, ts200.URL)++ results, err := s.FromData(context.Background(), true, []byte(verifyInput))+ if err != nil {+ t.Fatalf("FromData error: %v", err)+ }+ if len(results) != 1 {+ t.Fatalf("expected 1 result, got %d", len(results))+ }++ r := results[0]+ if !r.Verified {+ t.Error("expected Verified = true after second endpoint succeeded")+ }+ if r.VerificationError() != nil {+ t.Errorf("stale verification error not cleared: %v", r.VerificationError())+ }+}++// Verify that a clean 401 from the wrong region does not erase a transient+// error from an earlier endpoint. The verification error should survive so+// consumers know the result is uncertain, not definitively "not valid."+func TestSumoLogicKey_Verification_ErrorPreservedAcross401(t *testing.T) {+ ts500 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusInternalServerError)+ }))+ defer ts500.Close()++ ts401 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusUnauthorized)+ }))+ defer ts401.Close()++ s := Scanner{}+ _ = s.SetConfiguredEndpoints(ts500.URL, ts401.URL)++ results, err := s.FromData(context.Background(), true, []byte(verifyInput))+ if err != nil {+ t.Fatalf("FromData error: %v", err)+ }+ if len(results) != 1 {+ t.Fatalf("expected 1 result, got %d", len(results))+ }++ r := results[0]+ if r.Verified {+ t.Error("expected Verified = false")+ }+ if r.VerificationError() == nil {+ t.Error("expected verification error to be preserved after 401 from another endpoint, got nil")+ }+}