Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Improved JIRA detector (#4155)
Kashif Khanacb9826e007d
diff --git a/pkg/detectors/jiratoken/v1/jiratoken.go b/pkg/detectors/jiratoken/v1/jiratoken.goindex d6e4b83fa..fd94da957 100644--- a/pkg/detectors/jiratoken/v1/jiratoken.go+++ b/pkg/detectors/jiratoken/v1/jiratoken.go@@ -1,9 +1,11 @@package jiratokenimport (+ "bytes""context"- b64 "encoding/base64"+ "encoding/json""fmt"+ "io""net/http""strings"@@ -29,20 +31,22 @@ var (defaultClient = detectors.DetectorHttpClientWithLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.- tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)- domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b((?:[a-zA-Z0-9-]{1,24}\.)+[a-zA-Z0-9-]{2,24}\.[a-zA-Z0-9-]{2,16})\b`)- emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + common.EmailPattern)+ tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "confluence", "jira"}) + `\b([a-zA-Z-0-9]{24})\b`)+ domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "confluence", "jira"}) + `\b((?:[a-zA-Z0-9-]{1,24}\.)+[a-zA-Z0-9-]{2,24}\.[a-zA-Z0-9-]{2,16})\b`)+ emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "confluence", "jira"}) + common.EmailPattern))-const (- failedAuth = "AUTHENTICATED_FAILED"- loginReasonHeaderKey = "X-Seraph-LoginReason"-)+func (s Scanner) getClient() *http.Client {+ if s.client != nil {+ return s.client+ }+ return defaultClient+}// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {- return []string{"jira"}+ return []string{"atlassian", "confluence", "jira"}}// FromData will find and optionally verify JiraToken secrets in a given set of bytes.@@ -63,6 +67,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultuniqueEmails[strings.ToLower(email[1])] = struct{}{}}+ if len(uniqueDomains) == 0 {+ // reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943+ // In case we don't find any domain matches we can use this as the graphql API works with this domain if our authentication is valid+ uniqueDomains["api.atlassian.com"] = struct{}{}+ }+for email := range uniqueEmails {for token := range uniqueTokens {for domain := range uniqueDomains {@@ -78,7 +88,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif verify {client := s.getClient()- isVerified, verificationErr := verifyJiratoken(ctx, client, email, domain, token)+ isVerified, verificationErr := VerifyJiraToken(ctx, client, email, domain, token)s1.Verified = isVerifieds1.SetVerificationError(verificationErr, token)}@@ -91,40 +101,47 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) getClient() *http.Client {- if s.client != nil {- return s.client+func VerifyJiraToken(ctx context.Context, client *http.Client, email, domain, token string) (bool, error) {+ // wrap the query in a JSON body+ body := map[string]string{+ "query": `verify { me { user {name}}}`,}- return defaultClient-}-func verifyJiratoken(ctx context.Context, client *http.Client, email, domain, token string) (bool, error) {- data := fmt.Sprintf("%s:%s", email, token)- sEnc := b64.StdEncoding.EncodeToString([]byte(data))- req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domain+"/rest/api/3/dashboard", nil)+ // encode the body as JSON+ jsonBody, err := json.Marshal(body)if err != nil {return false, err}- req.Header.Add("Accept", "application/json")- req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))- res, err := client.Do(req)++ // api docs: https://developer.atlassian.com/platform/atlassian-graphql-api/graphql/#authentication+ req, err := http.NewRequestWithContext(ctx, http.MethodPost, "https://"+domain+"/gateway/api/graphql", bytes.NewBuffer(jsonBody))if err != nil {return false, err}- defer res.Body.Close()- // If the request is successful and the login reason is not failed authentication, then the token is valid.- // This is because Jira returns a 200 status code even if the token is invalid.- // Jira returns a default dashboard page.- if !(res.StatusCode >= 200 && res.StatusCode < 300) {- return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)+ req.Header.Add("Accept", "application/json")+ req.Header.Add("Content-Type", "application/json")+ req.SetBasicAuth(email, token)++ resp, err := client.Do(req)+ if err != nil {+ return false, err}- if res.Header.Get(loginReasonHeaderKey) != failedAuth {+ defer func() {+ _, _ = io.Copy(io.Discard, resp.Body)+ _ = resp.Body.Close()+ }()++ // the API returns 200 if the token is valid+ switch resp.StatusCode {+ case http.StatusOK:return true, nil+ case http.StatusUnauthorized:+ return false, nil+ default:+ return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)}-- return false, nil}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go b/pkg/detectors/jiratoken/v1/jiratoken_integration_test.goindex c4990b385..aa6db9b8e 100644--- a/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go+++ b/pkg/detectors/jiratoken/v1/jiratoken_integration_test.go@@ -154,7 +154,7 @@ func TestJiraToken_FromChunk(t *testing.T) {t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2.go b/pkg/detectors/jiratoken/v2/jiratoken_v2.goindex ca68806bf..ca300a6c8 100644--- a/pkg/detectors/jiratoken/v2/jiratoken_v2.go+++ b/pkg/detectors/jiratoken/v2/jiratoken_v2.go@@ -2,14 +2,15 @@ package jiratokenimport ("context"- b64 "encoding/base64""fmt""net/http""strings"regexp "github.com/wasilibs/go-re2"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ v1 "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/jiratoken/v1""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -29,45 +30,55 @@ var (// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/// Tokens created after Jan 18 2023 use a variable length- tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`)- domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b((?:[a-zA-Z0-9-]{1,24}\.)+[a-zA-Z0-9-]{2,24}\.[a-zA-Z0-9-]{2,16})\b`)- emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,})\b`)+ tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "confluence", "jira"}) + `\b(ATATT[A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`)+ domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "confluence", "jira"}) + `\b((?:[a-zA-Z0-9-]{1,24}\.)+[a-zA-Z0-9-]{2,24}\.[a-zA-Z0-9-]{2,16})\b`)+ emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"atlassian", "confluence", "jira"}) + common.EmailPattern))-const (- failedAuth = "AUTHENTICATED_FAILED"- loginReasonHeaderKey = "X-Seraph-LoginReason"-)+func (s Scanner) getClient() *http.Client {+ if s.client != nil {+ return s.client+ }+ return defaultClient+}// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {- return []string{"jira"}+ return []string{"atlassian", "confluence", "jira"}}// FromData will find and optionally verify JiraToken secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)- tokens := tokenPat.FindAllStringSubmatch(dataStr, -1)- domains := domainPat.FindAllStringSubmatch(dataStr, -1)- emails := emailPat.FindAllStringSubmatch(dataStr, -1)+ var uniqueTokens, uniqueDomains, uniqueEmails = make(map[string]struct{}), make(map[string]struct{}), make(map[string]struct{})- for _, email := range emails {- if len(email) != 2 {- continue- }- resEmail := strings.TrimSpace(email[1])+ for _, token := range tokenPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueTokens[token[1]] = struct{}{}+ }++ for _, domain := range domainPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueDomains[domain[1]] = struct{}{}+ }- for _, token := range tokens {- resToken := strings.TrimSpace(token[1])- for _, domain := range domains {- resDomain := strings.TrimSpace(domain[1])+ for _, email := range emailPat.FindAllStringSubmatch(dataStr, -1) {+ uniqueEmails[strings.ToLower(email[1])] = struct{}{}+ }+ if len(uniqueDomains) == 0 {+ // reason: https://community.atlassian.com/forums/Jira-Product-Discovery-questions/Authorization-issues-with-GRAPHQL/qaq-p/2640943+ // In case we don't find any domain matches we can use this as the graphql API works with this domain if our authentication is valid+ uniqueDomains["api.atlassian.com"] = struct{}{}+ }++ for email := range uniqueEmails {+ for token := range uniqueTokens {+ for domain := range uniqueDomains {s1 := detectors.Result{DetectorType: detectorspb.DetectorType_JiraToken,- Raw: []byte(resToken),- RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),+ Raw: []byte(token),+ RawV2: []byte(fmt.Sprintf("%s:%s:%s", email, token, domain)),ExtraData: map[string]string{"rotation_guide": "https://howtorotate.com/docs/tutorials/atlassian/","version": fmt.Sprintf("%d", s.Version()),@@ -76,9 +87,9 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif verify {client := s.getClient()- isVerified, verificationErr := verifyJiratoken(ctx, client, resEmail, resDomain, resToken)+ isVerified, verificationErr := v1.VerifyJiraToken(ctx, client, email, domain, token)s1.Verified = isVerified- s1.SetVerificationError(verificationErr, resToken)+ s1.SetVerificationError(verificationErr, token)}results = append(results, s1)@@ -89,42 +100,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}-func (s Scanner) getClient() *http.Client {- if s.client != nil {- return s.client- }- return defaultClient-}--func verifyJiratoken(ctx context.Context, client *http.Client, email, domain, token string) (bool, error) {- data := fmt.Sprintf("%s:%s", email, token)- sEnc := b64.StdEncoding.EncodeToString([]byte(data))- req, err := http.NewRequestWithContext(ctx, "GET", "https://"+domain+"/rest/api/3/dashboard", nil)- if err != nil {- return false, err- }- req.Header.Add("Accept", "application/json")- req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))- res, err := client.Do(req)- if err != nil {- return false, err- }- defer res.Body.Close()-- // If the request is successful and the login reason is not failed authentication, then the token is valid.- // This is because Jira returns a 200 status code even if the token is invalid.- // Jira returns a default dashboard page.- if !(res.StatusCode >= 200 && res.StatusCode < 300) {- return false, fmt.Errorf("unexpected HTTP response status %d", res.StatusCode)- }-- if res.Header.Get(loginReasonHeaderKey) != failedAuth {- return true, nil- }-- return false, nil-}-func (s Scanner) Type() detectorspb.DetectorType {return detectorspb.DetectorType_JiraToken}diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go b/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.goindex 92b248dba..6bf2ee3b2 100644--- a/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go+++ b/pkg/detectors/jiratoken/v2/jiratoken_v2_integration_test.go@@ -154,7 +154,7 @@ func TestJiraToken_FromChunk(t *testing.T) {t.Fatalf("wantVerificationError = %v, verification error = %v", tt.wantVerificationErr, got[i].VerificationError())}}- ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError")+ ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "Raw", "RawV2", "verificationError", "primarySecret")if diff := cmp.Diff(got, tt.want, ignoreOpts); diff != "" {t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)}diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go b/pkg/detectors/jiratoken/v2/jiratoken_v2_test.goindex 4c8db5467..cbf1e8ad9 100644--- a/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go+++ b/pkg/detectors/jiratoken/v2/jiratoken_v2_test.go@@ -3,6 +3,7 @@ package jiratokenimport ("context""fmt"+ "strings""testing""github.com/google/go-cmp/cmp"@@ -12,7 +13,7 @@ import ()var (- validTokenPattern = "9nsCADa7812Z7VoIsYJ0K4rFWLBfk=1rhOsLAW"+ validTokenPattern = "ATATT9nsCADa7812Z7VoIsYJ0K4rFWLBfk=1rhOsLAW"invalidTokenPattern = "9nsCA?a7812Z7VoI%YJ0K4rFWLBfk91rhOsLAW"validDomainPattern = "hereisavalidsubdomain.heresalongdomain.com"validDomainPattern2 = "jira.hereisavalidsubdomain.heresalongdomain.com"@@ -33,12 +34,12 @@ func TestJiraToken_Pattern(t *testing.T) {{name: "valid pattern - with keyword jira",input: fmt.Sprintf("%s %s \n%s %s\n%s %s", keyword, validTokenPattern, keyword, validDomainPattern, keyword, validEmailPattern),- want: []string{validEmailPattern + ":" + validTokenPattern + ":" + validDomainPattern},+ want: []string{strings.ToLower(validEmailPattern) + ":" + validTokenPattern + ":" + validDomainPattern},},{name: "valid pattern - with multiple subdomains",input: fmt.Sprintf("%s %s \n%s %s\n%s %s", keyword, validTokenPattern, keyword, validDomainPattern2, keyword, validEmailPattern),- want: []string{validEmailPattern + ":" + validTokenPattern + ":" + validDomainPattern2},+ want: []string{strings.ToLower(validEmailPattern) + ":" + validTokenPattern + ":" + validDomainPattern2},},{name: "valid pattern - key out of prefix range",