Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
fix(jiratoken): stop 202 reverification retry loop (#5200)
Ahsan-Sarbaz96b593f460ba
diff --git a/pkg/detectors/jiratoken/v1/jiratoken.go b/pkg/detectors/jiratoken/v1/jiratoken.goindex b63b132f3..65027e261 100644--- a/pkg/detectors/jiratoken/v1/jiratoken.go+++ b/pkg/detectors/jiratoken/v1/jiratoken.go@@ -8,6 +8,9 @@ import ("fmt""io""net/http"+ "net/url"+ "slices"+ "sort""strings"regexp "github.com/wasilibs/go-re2"@@ -20,15 +23,25 @@ import (type Scanner struct {detectors.DefaultMultiPartCredentialProvider+ detectors.EndpointSetterclient *http.Client}// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)var _ detectors.Versioner = (*Scanner)(nil)+var _ detectors.EndpointCustomizer = (*Scanner)(nil)+var _ detectors.CloudProvider = (*Scanner)(nil)func (Scanner) Version() int { return 1 }+// CloudHost is used whenever the scanned data carries no Atlassian host of its own.+// reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943+// The graphql API answers on this host as long as the credentials are valid.+const CloudHost = "api.atlassian.com"++func (Scanner) CloudEndpoint() string { return "https://" + CloudHost }+var (defaultClient = detectors.DetectorHttpClientWithLocalAddresses@@ -40,8 +53,79 @@ var (invalidHosts = simple.NewCache[struct{}]()errNoHost = errors.New("no such host")++ // atlassianDomains are the only hosts that can answer the verification request.+ // domainPat matches any domain-shaped string near an atlassian/jira/confluence keyword, so without+ // this filter the detector would send the email and token to unrelated hosts found in the same chunk.+ atlassianDomains = []string{"atlassian.net", "atlassian.com", "jira.com"})+// IsAtlassianHost reports whether host is owned by Atlassian and can therefore be used for verification.+func IsAtlassianHost(host string) bool {+ host = strings.ToLower(strings.TrimSuffix(host, "."))+ for _, domain := range atlassianDomains {+ if host == domain || strings.HasSuffix(host, "."+domain) {+ return true+ }+ }+ return false+}++// EndpointHost reduces an endpoint to its bare host. Found domains arrive without a scheme while cloud and+// user-configured endpoints carry one, and the analyzer expects SecretParts["domain"] to be a bare host.+func EndpointHost(endpoint string) string {+ endpoint = strings.TrimSuffix(strings.TrimSpace(endpoint), "/")+ if u, err := url.Parse(endpoint); err == nil && u.Host != "" {+ return u.Host+ }+ return endpoint+}++// FoundHosts returns the Atlassian hosts among the domains matched in the data, sorted so results are+// deterministic. domainPat matches any domain-shaped string, so everything else is dropped rather than+// being sent the email and token.+func FoundHosts(domains map[string]struct{}) []string {+ hosts := make([]string, 0, len(domains))+ for domain := range domains {+ if IsAtlassianHost(domain) {+ hosts = append(hosts, domain)+ }+ }+ sort.Strings(hosts)+ return hosts+}++// VerificationHosts reduces endpoints to the bare hosts to verify against, dropping duplicates. The+// analyzer expects SecretParts["domain"] to be a bare host, and a user-configured endpoint can repeat a+// found one.+//+// CloudHost is only a fallback for data that named no Atlassian host, and EndpointSetter places it ahead+// of the found hosts, so it is dropped when the data named hosts of its own. Data that named CloudHost+// itself keeps it, since then it is a found host rather than the fallback.+func VerificationHosts(endpoints, found []string) []string {+ seen := make(map[string]struct{}, len(endpoints))+ hosts := make([]string, 0, len(endpoints))+ dropCloud := len(found) > 0 && !slices.Contains(found, CloudHost)++ for _, endpoint := range endpoints {+ host := EndpointHost(endpoint)+ if host == "" || (dropCloud && host == CloudHost) {+ continue+ }+ if _, ok := seen[host]; ok {+ continue+ }+ seen[host] = struct{}{}+ hosts = append(hosts, host)+ }+ return hosts+}++// verificationURL builds the graphql URL for a bare host such as "acme.atlassian.net".+func verificationURL(host string) string {+ return "https://" + host + "/gateway/api/graphql"+}+type JIRAGraphQLResponse struct {Data struct {Me struct {@@ -83,24 +167,20 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultuniqueEmails[strings.ToLower(email[1])] = struct{}{}}- if len(uniqueDomains) == 0 {- // reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943- // In case we don't find any domain matches we can use this as the graphql API works with this domain if our authentication is valid- uniqueDomains["api.atlassian.com"] = struct{}{}- }+ found := FoundHosts(uniqueDomains)+ hosts := VerificationHosts(s.Endpoints(found...), found)for email := range uniqueEmails {for token := range uniqueTokens {- for domain := range uniqueDomains {+ for _, domain := range hosts {if invalidHosts.Exists(domain) {- delete(uniqueDomains, domain)continue}s1 := detectors.Result{DetectorType: detector_typepb.DetectorType_JiraToken,Raw: []byte(token),- RawV2: []byte(fmt.Sprintf("%s:%s:%s", email, token, domain)),+ RawV2: fmt.Appendf(nil, "%s:%s:%s", email, token, domain),ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/","version": fmt.Sprintf("%d", s.Version()),@@ -123,6 +203,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1.SetVerificationError(verificationErr, token)}++ // The credential is the same across endpoints, so once one of them verifies it there+ // is nothing to learn from the rest.+ if s1.Verified {+ results = append(results, s1)+ break+ }}results = append(results, s1)@@ -146,7 +233,7 @@ func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, to}// api docs: https://developer.atlassian.com/platform/atlassian-graphql-api/graphql/#authentication- req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://"+domain+"/gateway/api/graphql", bytes.NewBuffer(jsonBody))+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, verificationURL(domain), bytes.NewBuffer(jsonBody))if err != nil {return false, err}@@ -171,8 +258,8 @@ func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, to}()// the API returns 200 if the token is valid- switch resp.StatusCode {- case http.StatusOK:+ switch {+ case resp.StatusCode == http.StatusOK:var jiraResp JIRAGraphQLResponseif err := json.NewDecoder(resp.Body).Decode(&jiraResp); err != nil {return false, nil // can't decode response in case of 200 OK = not valid JIRA domain@@ -181,10 +268,19 @@ func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, to// A 200 on its own isn't enough - unrelated hosts can also return JSON and get flagged as verified.// Only trust it when the body actually carries the authenticated user's name.return jiraResp.Data.Me.User.Name != "", nil- case http.StatusUnauthorized:+ case resp.StatusCode == http.StatusUnauthorized, resp.StatusCode == http.StatusForbidden:+ // The credentials were rejected. Nothing to retry.return false, nil- default:+ case resp.StatusCode == http.StatusTooManyRequests, resp.StatusCode >= 500:+ // Rate limiting and server errors are genuinely transient, so keep them indeterminate+ // and let the secret be retried.return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)+ default:+ // Anything else (202, 3xx, 404, ...) is not an Atlassian authentication response at all.+ // domainPat matches any domain-shaped string near a jira/atlassian/confluence keyword, so this+ // request may well have gone to an unrelated host that happily accepts POSTs. Retrying will+ // return the same status forever, so treat it as a terminal "not verified" instead of an error.+ return false, nil}}diff --git a/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go b/pkg/detectors/jiratoken/v1/jiratoken_integration_test.goindex 8eb9252d7..50cc391fa 100644--- a/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go+++ b/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go@@ -118,7 +118,7 @@ func TestJiraToken_FromChunk(t *testing.T) {wantVerificationErr: true,},{- name: "found, verified but unexpected api surface",+ name: "found, unexpected api surface is terminal and not an error",s: Scanner{client: common.ConstantResponseHttpClient(404, "")},args: args{ctx: context.Background(),@@ -136,11 +136,14 @@ func TestJiraToken_FromChunk(t *testing.T) {},},wantErr: false,- wantVerificationErr: true,+ wantVerificationErr: false,},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {+ tt.s.SetCloudEndpoint(tt.s.CloudEndpoint())+ tt.s.UseCloudEndpoint(true)+ tt.s.UseFoundEndpoints(true)got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)if (err != nil) != tt.wantErr {t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)@@ -165,6 +168,9 @@ func TestJiraToken_FromChunk(t *testing.T) {func BenchmarkFromData(benchmark *testing.B) {ctx := context.Background()s := Scanner{}+ s.SetCloudEndpoint(s.CloudEndpoint())+ s.UseCloudEndpoint(true)+ s.UseFoundEndpoints(true)for name, data := range detectors.MustGetBenchmarkData() {benchmark.Run(name, func(b *testing.B) {b.ResetTimer()diff --git a/pkg/detectors/jiratoken/v1/jiratoken_test.go b/pkg/detectors/jiratoken/v1/jiratoken_test.goindex c61fd8572..cba80e7e2 100644--- a/pkg/detectors/jiratoken/v1/jiratoken_test.go+++ b/pkg/detectors/jiratoken/v1/jiratoken_test.go@@ -15,7 +15,9 @@ import (var (validTokenPattern = "Z7VoIYJ0K4rFWLBfkhOsLAWX"invalidTokenPattern = "Z7VoI?J0K4rF#LBfkhO&LAWX"- validDomainPattern = "hereisavalidsubdomain.heresalongdomain.com"+ validDomainPattern = "hereisavalidsubdomain.atlassian.net"+ foreignDomainPattern = "hereisavalidsubdomain.heresalongdomain.com"+ cloudDomain = "api.atlassian.com"invalidDomainPattern = "?y4r3fs1ewqec12v1e3tl.5Hcsrcehic89saXd.ro@"validEmailPattern = "[email protected]"invalidEmailPattern = "xfKF_BZq7/grum.com"@@ -24,6 +26,9 @@ var (func TestJiraToken_Pattern(t *testing.T) {d := Scanner{}+ d.SetCloudEndpoint(d.CloudEndpoint())+ d.UseCloudEndpoint(true)+ d.UseFoundEndpoints(true)ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {name string@@ -33,7 +38,16 @@ func TestJiraToken_Pattern(t *testing.T) {{name: "valid pattern - with keyword jira",input: fmt.Sprintf("%s %s \n%s %s\n%s %s", keyword, validTokenPattern, keyword, validDomainPattern, keyword, validEmailPattern),- want: []string{validEmailPattern + ":" + validTokenPattern + ":" + validDomainPattern},+ // The tenant host found in the data is used on its own. The cloud endpoint must not shadow it,+ // or SecretParts["domain"] would stop being the host the analyzer needs.+ want: []string{validEmailPattern + ":" + validTokenPattern + ":" + validDomainPattern},+ },+ {+ // A domain that isn't Atlassian-owned can't answer the verification request, so it is dropped+ // and only the cloud endpoint remains.+ name: "non-atlassian domain falls back to the cloud endpoint",+ input: fmt.Sprintf("%s %s \n%s %s\n%s %s", keyword, validTokenPattern, keyword, foreignDomainPattern, keyword, validEmailPattern),+ want: []string{validEmailPattern + ":" + validTokenPattern + ":" + cloudDomain},},{name: "valid pattern - key out of prefix range",@@ -90,12 +104,121 @@ func TestJiraToken_Pattern(t *testing.T) {}}+func TestIsAtlassianHost(t *testing.T) {+ tests := map[string]bool{+ "example.atlassian.net": true,+ "atlassian.net": true,+ "api.atlassian.com": true,+ "acme.jira.com": true,+ "EXAMPLE.Atlassian.Net": true,+ "example.atlassian.net.": true,+ // Suffix lookalikes must not pass, or the filter would be trivially bypassed.+ "notatlassian.net": false,+ "atlassian.net.evil.com": false,+ "example.com": false,+ "hereisavalidsubdomain.co.uk": false,+ }++ for host, want := range tests {+ t.Run(host, func(t *testing.T) {+ if got := IsAtlassianHost(host); got != want {+ t.Errorf("IsAtlassianHost(%q) = %v, want %v", host, got, want)+ }+ })+ }+}++func TestFoundHosts(t *testing.T) {+ tests := []struct {+ name string+ domains []string+ want []string+ }{+ {+ name: "non-atlassian domains are dropped",+ domains: []string{"tenant.atlassian.net", "hooks.example.com"},+ want: []string{"tenant.atlassian.net"},+ },+ {+ name: "nothing atlassian found",+ domains: []string{"hooks.example.com"},+ want: []string{},+ },+ {+ // The cloud host is a normal Atlassian host when it shows up in the data.+ name: "cloud host in the data is kept",+ domains: []string{CloudHost},+ want: []string{CloudHost},+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ domains := make(map[string]struct{}, len(test.domains))+ for _, d := range test.domains {+ domains[d] = struct{}{}+ }+ if diff := cmp.Diff(test.want, FoundHosts(domains)); diff != "" {+ t.Errorf("FoundHosts() diff: (-want +got)\n%s", diff)+ }+ })+ }+}++func TestVerificationHosts(t *testing.T) {+ const cloudEndpoint = "https://" + CloudHost++ tests := []struct {+ name string+ endpoints []string+ found []string+ want []string+ }{+ {+ // EndpointSetter puts the cloud endpoint ahead of the found ones, so leaving it in would let it+ // win the break-on-verified and report a domain the analyzer cannot use.+ name: "cloud dropped when the data named a host",+ endpoints: []string{cloudEndpoint, "tenant.atlassian.net"},+ found: []string{"tenant.atlassian.net"},+ want: []string{"tenant.atlassian.net"},+ },+ {+ name: "cloud kept when the data named nothing",+ endpoints: []string{cloudEndpoint},+ found: nil,+ want: []string{CloudHost},+ },+ {+ // Dropping the cloud host here would leave no hosts at all and produce no results.+ name: "cloud kept when the data named it",+ endpoints: []string{cloudEndpoint, CloudHost},+ found: []string{CloudHost},+ want: []string{CloudHost},+ },+ {+ name: "configured endpoints are reduced to hosts and deduped",+ endpoints: []string{"https://jira.example.com:8443/", cloudEndpoint, "tenant.atlassian.net", "https://tenant.atlassian.net", ""},+ found: []string{"tenant.atlassian.net"},+ want: []string{"jira.example.com:8443", "tenant.atlassian.net"},+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ if diff := cmp.Diff(test.want, VerificationHosts(test.endpoints, test.found)); diff != "" {+ t.Errorf("VerificationHosts() diff: (-want +got)\n%s", diff)+ }+ })+ }+}+func TestJiraToken_Verify(t *testing.T) {tests := []struct {name stringstatusCode intbody stringwantVerified bool+ wantErr bool}{{name: "authenticated user in body is verified",@@ -116,14 +239,48 @@ func TestJiraToken_Verify(t *testing.T) {body: `{"code":401,"message":"Unauthorized"}`,wantVerified: false,},+ {+ name: "forbidden is not verified",+ statusCode: 403,+ body: `{"code":403,"message":"Forbidden"}`,+ wantVerified: false,+ },+ {+ // Atlassian never answers 202 here. It comes from unrelated hosts that accept the POST, and it+ // would repeat on every retry, so it has to be terminal rather than an indeterminate error.+ name: "accepted is not verified and not an error",+ statusCode: 202,+ body: `{"status":"accepted"}`,+ wantVerified: false,+ },+ {+ name: "not found is not verified and not an error",+ statusCode: 404,+ body: `{"code":404,"message":"Not Found"}`,+ wantVerified: false,+ },+ {+ name: "rate limited is indeterminate",+ statusCode: 429,+ body: `{"code":429,"message":"Too Many Requests"}`,+ wantVerified: false,+ wantErr: true,+ },+ {+ name: "server error is indeterminate",+ statusCode: 500,+ body: `{"code":500,"message":"Internal Server Error"}`,+ wantVerified: false,+ wantErr: true,+ },}for _, test := range tests {t.Run(test.name, func(t *testing.T) {client := common.ConstantResponseHttpClient(test.statusCode, test.body)verified, err := VerifyJiraToken(context.Background(), client, "[email protected]", "example.atlassian.net", validTokenPattern)- if err != nil {- t.Fatalf("unexpected error: %v", err)+ if (err != nil) != test.wantErr {+ t.Fatalf("got error = %v, wantErr %v", err, test.wantErr)}if verified != test.wantVerified {t.Errorf("got verified = %v, want %v", verified, test.wantVerified)diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2.go b/pkg/detectors/jiratoken/v2/jiratoken_v2.goindex 6e802e0c9..90e275d7e 100644--- a/pkg/detectors/jiratoken/v2/jiratoken_v2.go+++ b/pkg/detectors/jiratoken/v2/jiratoken_v2.go@@ -17,14 +17,19 @@ import (type Scanner struct {client *http.Clientdetectors.DefaultMultiPartCredentialProvider+ detectors.EndpointSetter}// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)var _ detectors.Versioner = (*Scanner)(nil)+var _ detectors.EndpointCustomizer = (*Scanner)(nil)+var _ detectors.CloudProvider = (*Scanner)(nil)func (Scanner) Version() int { return 2 }+func (Scanner) CloudEndpoint() string { return "https://" + v1.CloudHost }+var (defaultClient = detectors.DetectorHttpClientWithLocalAddresses@@ -66,19 +71,17 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultuniqueEmails[strings.ToLower(email[1])] = struct{}{}}- if len(uniqueDomains) == 0 {- // reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943- // In case we don't find any domain matches we can use this as the graphql API works with this domain if our authentication is valid- uniqueDomains["api.atlassian.com"] = struct{}{}- }+ // domainPat here is not even keyword-gated, so it captures every domain in the chunk.+ found := v1.FoundHosts(uniqueDomains)+ hosts := v1.VerificationHosts(s.Endpoints(found...), found)for email := range uniqueEmails {for token := range uniqueTokens {- for domain := range uniqueDomains {+ for _, domain := range hosts {s1 := detectors.Result{DetectorType: detector_typepb.DetectorType_JiraToken,Raw: []byte(token),- RawV2: []byte(fmt.Sprintf("%s:%s:%s", email, token, domain)),+ RawV2: fmt.Appendf(nil, "%s:%s:%s", email, token, domain),ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/","version": fmt.Sprintf("%d", s.Version()),@@ -95,6 +98,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultisVerified, verificationErr := v1.VerifyJiraToken(ctx, client, email, domain, token)s1.Verified = isVerifieds1.SetVerificationError(verificationErr, token)++ // The credential is the same across endpoints, so once one of them verifies it there+ // is nothing to learn from the rest.+ if s1.Verified {+ results = append(results, s1)+ break+ }}results = append(results, s1)diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go b/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.goindex d72a7f331..1dca3b25b 100644--- a/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go+++ b/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go@@ -141,6 +141,9 @@ func TestJiraToken_FromChunk(t *testing.T) {}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {+ tt.s.SetCloudEndpoint(tt.s.CloudEndpoint())+ tt.s.UseCloudEndpoint(true)+ tt.s.UseFoundEndpoints(true)got, err := tt.s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)if (err != nil) != tt.wantErr {t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)@@ -165,6 +168,9 @@ func TestJiraToken_FromChunk(t *testing.T) {func BenchmarkFromData(benchmark *testing.B) {ctx := context.Background()s := Scanner{}+ s.SetCloudEndpoint(s.CloudEndpoint())+ s.UseCloudEndpoint(true)+ s.UseFoundEndpoints(true)for name, data := range detectors.MustGetBenchmarkData() {benchmark.Run(name, func(b *testing.B) {b.ResetTimer()diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go b/pkg/detectors/jiratoken/v2/jiratoken_v2_test.goindex 4249d15b5..40c5b8a42 100644--- a/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go+++ b/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go@@ -12,6 +12,9 @@ import (func TestJiraToken_Pattern(t *testing.T) {d := Scanner{}+ d.SetCloudEndpoint(d.CloudEndpoint())+ d.UseCloudEndpoint(true)+ d.UseFoundEndpoints(true)ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})tests := []struct {name string