Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
update jira detector to match new variable tokens (#1720)
Mike Vanbuskirk2e4b17d3f43d
diff --git a/hack/docs/Adding_Detectors_Internal.md b/hack/docs/Adding_Detectors_Internal.mdindex 4b0685bed..79db10353 100644--- a/hack/docs/Adding_Detectors_Internal.md+++ b/hack/docs/Adding_Detectors_Internal.md@@ -44,6 +44,18 @@ If you think that something should be included outside of these guidelines, plea- Go 1.17+- Make+### Adding New Token Formats to an Existing Scanner++In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.++1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`++Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.++2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)++3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)+### Creating a new Secret Scanner1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.@@ -96,7 +108,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets ManagerNote: `/tmp/s` is a valid path on Linux. You will need to change that for Windows or OSX, otherwise you will see an error. On Windows you will also need to install [WSL](https://docs.microsoft.com/en-us/windows/wsl/install).```bash- gcloud secrets versions access --project trufflehog-testing --secret detectors3 latest > /tmp/s+ gcloud secrets versions access --project trufflehog-testing --secret detectors5 latest > /tmp/s```2. Add the secret that you need for testing.@@ -113,8 +125,9 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager3. Update the secret version with your modification.```bash- gcloud secrets versions add --project trufflehog-testing detectors3 --data-file /tmp/s+ gcloud secrets versions add --project trufflehog-testing detectors5 --data-file /tmp/s```+ Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb).4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go).diff --git a/hack/docs/Adding_Detectors_external.md b/hack/docs/Adding_Detectors_external.mdindex fc9bf42e0..1ae072c4e 100644--- a/hack/docs/Adding_Detectors_external.md+++ b/hack/docs/Adding_Detectors_external.md@@ -40,6 +40,18 @@ If you think that something should be included outside of these guidelines, plea- Go 1.17+- Make+### Adding New Token Formats to an Existing Scanner++In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.++1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`++Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.++2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)++3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)+### Creating a new Secret Scanner1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.diff --git a/pkg/detectors/jiratoken/jiratoken.go b/pkg/detectors/jiratoken/jiratoken.goindex bfb2fc9eb..4b5ed368e 100644--- a/pkg/detectors/jiratoken/jiratoken.go+++ b/pkg/detectors/jiratoken/jiratoken.go@@ -17,6 +17,9 @@ type Scanner struct{}// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.Versioner = (*Scanner)(nil)++func (Scanner) Version() int { return 1 }var (client = common.SaneHttpClient()diff --git a/pkg/detectors/jiratoken/jiratoken_test.go b/pkg/detectors/jiratoken/jiratoken_test.goindex 67e5df3f7..460992e5d 100644--- a/pkg/detectors/jiratoken/jiratoken_test.go+++ b/pkg/detectors/jiratoken/jiratoken_test.go@@ -19,7 +19,7 @@ import (func TestJiraToken_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}diff --git a/pkg/detectors/jiratoken_v2/jiratoken_v2.go b/pkg/detectors/jiratoken_v2/jiratoken_v2.gonew file mode 100644index 000000000..f6c18e017--- /dev/null+++ b/pkg/detectors/jiratoken_v2/jiratoken_v2.go@@ -0,0 +1,118 @@+package jiratoken_v2++import (+ "context"+ b64 "encoding/base64"+ "fmt"+ "net/http"+ "regexp"+ "strings"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time.+var _ detectors.Detector = (*Scanner)(nil)+var _ detectors.Versioner = (*Scanner)(nil)++func (Scanner) Version() int { return 2 }++var (+ client = common.SaneHttpClient()++ // https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/+ // Tokens created after Jan 18 2023 use a variable length+ tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`)+ domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\.[a-zA-Z-0-9]{3,16}\.[a-zA-Z-0-9]{3,16})\b`)+ emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`)+)++const (+ failedAuth = "AUTHENTICATED_FAILED"+ loginReasonHeaderKey = "X-Seraph-LoginReason"+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"jira"}+}++// FromData will find and optionally verify JiraToken secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ tokens := tokenPat.FindAllStringSubmatch(dataStr, -1)+ domains := domainPat.FindAllStringSubmatch(dataStr, -1)+ emails := emailPat.FindAllStringSubmatch(dataStr, -1)++ for _, email := range emails {+ email = strings.Split(email[0], " ")+ if len(email) != 2 {+ continue+ }+ resEmail := strings.TrimSpace(email[1])+ for _, token := range tokens {+ if len(token) != 2 {+ continue+ }+ resToken := strings.TrimSpace(token[1])+ for _, domain := range domains {+ if len(domain) != 2 {+ continue+ }+ resDomain := strings.TrimSpace(domain[1])++ s1 := detectors.Result{+ DetectorType: detectorspb.DetectorType_JiraToken,+ Raw: []byte(resToken),+ RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),+ }++ if verify {+ data := fmt.Sprintf("%s:%s", resEmail, resToken)+ sEnc := b64.StdEncoding.EncodeToString([]byte(data))+ req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)+ if err != nil {+ continue+ }+ req.Header.Add("Accept", "application/json")+ req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))+ res, err := client.Do(req)+ if err == nil {+ defer res.Body.Close()++ // If the request is successful and the login reason is not failed authentication, then the token is valid.+ // This is because Jira returns a 200 status code even if the token is invalid.+ // Jira returns a default dashboard page.+ if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {+ s1.Verified = true+ } else {+ s1.VerificationError = err+ }+ }+ }++ if !s1.Verified {+ if detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {+ continue+ }+ }++ results = append(results, s1)++ }++ }+ }++ return results, nil+}++func (s Scanner) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType_JiraToken+}diff --git a/pkg/detectors/jiratoken_v2/jiratoken_v2_test.go b/pkg/detectors/jiratoken_v2/jiratoken_v2_test.gonew file mode 100644index 000000000..cdcc6e875--- /dev/null+++ b/pkg/detectors/jiratoken_v2/jiratoken_v2_test.go@@ -0,0 +1,123 @@+//go:build detectors+// +build detectors++package jiratoken_v2++import (+ "context"+ "fmt"+ "testing"+ "time"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"++ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestJiraToken_FromChunk(t *testing.T) {+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)+ defer cancel()+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")+ if err != nil {+ t.Fatalf("could not get test secrets from GCP: %s", err)+ }+ token := testSecrets.MustGetField("JIRA_V2_TOKEN")+ inactiveToken := testSecrets.MustGetField("JIRA_V2_INACTIVE")+ email := testSecrets.MustGetField("JIRA_V2_EMAIL")+ domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")++ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "found, verified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_JiraToken,+ Verified: true,+ },+ },+ wantErr: false,+ },+ {+ name: "found, unverified",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_JiraToken,+ Verified: false,+ },+ },+ wantErr: false,+ },+ {+ name: "not found",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("You cannot find the secret within"),+ verify: true,+ },+ want: nil,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ for i := range got {+ if len(got[i].Raw) == 0 {+ t.Fatalf("no raw secret present: \n %+v", got[i])+ }+ got[i].Raw = nil+ got[i].RawV2 = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ b.ResetTimer()+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}