Skip to content

Find, verify, and analyze leaked credentials

update jira detector to match new variable tokens (#1720)

Mike Vanbuskirk2e4b17d3f43d
diff --git a/hack/docs/Adding_Detectors_Internal.md b/hack/docs/Adding_Detectors_Internal.md
index 4b0685bed..79db10353 100644
--- a/hack/docs/Adding_Detectors_Internal.md
+++ b/hack/docs/Adding_Detectors_Internal.md
@@ -44,6 +44,18 @@ If you think that something should be included outside of these guidelines, plea
- Go 1.17+
- Make
+### Adding New Token Formats to an Existing Scanner
+
+In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
+
+1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
+
+Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
+
+2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
+
+3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
+
### Creating a new Secret Scanner
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum.
@@ -96,7 +108,7 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
Note: `/tmp/s` is a valid path on Linux. You will need to change that for Windows or OSX, otherwise you will see an error. On Windows you will also need to install [WSL](https://docs.microsoft.com/en-us/windows/wsl/install).
```bash
- gcloud secrets versions access --project trufflehog-testing --secret detectors3 latest > /tmp/s
+ gcloud secrets versions access --project trufflehog-testing --secret detectors5 latest > /tmp/s
```
2. Add the secret that you need for testing.
@@ -113,8 +125,9 @@ Do not embed test credentials in the test code. Instead, use GCP Secrets Manager
3. Update the secret version with your modification.
```bash
- gcloud secrets versions add --project trufflehog-testing detectors3 --data-file /tmp/s
+ gcloud secrets versions add --project trufflehog-testing detectors5 --data-file /tmp/s
```
+ Note: We increment the detectors file name `detectors(n+1)` once the previous one exceeds the max size allowed by GSM (65kb).
4. Access the secret value as shown in the [example code](pkg/detectors/heroku/heroku_test.go).
diff --git a/hack/docs/Adding_Detectors_external.md b/hack/docs/Adding_Detectors_external.md
index fc9bf42e0..1ae072c4e 100644
--- a/hack/docs/Adding_Detectors_external.md
+++ b/hack/docs/Adding_Detectors_external.md
@@ -40,6 +40,18 @@ If you think that something should be included outside of these guidelines, plea
- Go 1.17+
- Make
+### Adding New Token Formats to an Existing Scanner
+
+In some instances, services will update their token format, requiring a new regex to properly detect secrets in addition to supporting the previous token format. Accomodating this can be done without adding a net-new detector. [We provide a `Versioner` interface](https://github.com/trufflesecurity/trufflehog/blob/e18cfd5e0af1469a9f05b8d5732bcc94c39da49c/pkg/detectors/detectors.go#L30) that can be implemented.
+
+1. Create a copy of the package and append `_v2` to the package and file names. Ex: `<packagename>/` -> `<packagename>_v2`, `<packagename>.go` -> `<packagename>_v2.go`
+
+Note: Be sure to update the tests to reference the new secret values in GSM, or the tests will fail.
+
+2. Implement the `Versioner` interface. [GitHub example implementation.](/pkg/detectors/github_old/github_old.go#L22)
+
+3. Proceed from step 3 of [Creating a new Secret Scanner](#creating-a-new-secret-scanner)
+
### Creating a new Secret Scanner
1. Identify the Secret Detector name from the [/proto/detectors.proto](/proto/detectors.proto) `DetectorType` enum. If necessary, run `make protos` when adding new ones.
diff --git a/pkg/detectors/jiratoken/jiratoken.go b/pkg/detectors/jiratoken/jiratoken.go
index bfb2fc9eb..4b5ed368e 100644
--- a/pkg/detectors/jiratoken/jiratoken.go
+++ b/pkg/detectors/jiratoken/jiratoken.go
@@ -17,6 +17,9 @@ type Scanner struct{}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (Scanner) Version() int { return 1 }
var (
client = common.SaneHttpClient()
diff --git a/pkg/detectors/jiratoken/jiratoken_test.go b/pkg/detectors/jiratoken/jiratoken_test.go
index 67e5df3f7..460992e5d 100644
--- a/pkg/detectors/jiratoken/jiratoken_test.go
+++ b/pkg/detectors/jiratoken/jiratoken_test.go
@@ -19,7 +19,7 @@ import (
func TestJiraToken_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors3")
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
diff --git a/pkg/detectors/jiratoken_v2/jiratoken_v2.go b/pkg/detectors/jiratoken_v2/jiratoken_v2.go
new file mode 100644
index 000000000..f6c18e017
--- /dev/null
+++ b/pkg/detectors/jiratoken_v2/jiratoken_v2.go
@@ -0,0 +1,118 @@
+package jiratoken_v2
+
+import (
+ "context"
+ b64 "encoding/base64"
+ "fmt"
+ "net/http"
+ "regexp"
+ "strings"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type Scanner struct{}
+
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*Scanner)(nil)
+var _ detectors.Versioner = (*Scanner)(nil)
+
+func (Scanner) Version() int { return 2 }
+
+var (
+ client = common.SaneHttpClient()
+
+ // https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
+ // Tokens created after Jan 18 2023 use a variable length
+ tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([A-Za-z0-9+/=_-]+=[A-Za-z0-9]{8})\b`)
+ domainPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{5,24}\.[a-zA-Z-0-9]{3,16}\.[a-zA-Z-0-9]{3,16})\b`)
+ emailPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Z|a-z]{2,}\b`)
+)
+
+const (
+ failedAuth = "AUTHENTICATED_FAILED"
+ loginReasonHeaderKey = "X-Seraph-LoginReason"
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s Scanner) Keywords() []string {
+ return []string{"jira"}
+}
+
+// FromData will find and optionally verify JiraToken secrets in a given set of bytes.
+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+
+ tokens := tokenPat.FindAllStringSubmatch(dataStr, -1)
+ domains := domainPat.FindAllStringSubmatch(dataStr, -1)
+ emails := emailPat.FindAllStringSubmatch(dataStr, -1)
+
+ for _, email := range emails {
+ email = strings.Split(email[0], " ")
+ if len(email) != 2 {
+ continue
+ }
+ resEmail := strings.TrimSpace(email[1])
+ for _, token := range tokens {
+ if len(token) != 2 {
+ continue
+ }
+ resToken := strings.TrimSpace(token[1])
+ for _, domain := range domains {
+ if len(domain) != 2 {
+ continue
+ }
+ resDomain := strings.TrimSpace(domain[1])
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_JiraToken,
+ Raw: []byte(resToken),
+ RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
+ }
+
+ if verify {
+ data := fmt.Sprintf("%s:%s", resEmail, resToken)
+ sEnc := b64.StdEncoding.EncodeToString([]byte(data))
+ req, err := http.NewRequestWithContext(ctx, "GET", "https://"+resDomain+"/rest/api/3/dashboard", nil)
+ if err != nil {
+ continue
+ }
+ req.Header.Add("Accept", "application/json")
+ req.Header.Add("Authorization", fmt.Sprintf("Basic %s", sEnc))
+ res, err := client.Do(req)
+ if err == nil {
+ defer res.Body.Close()
+
+ // If the request is successful and the login reason is not failed authentication, then the token is valid.
+ // This is because Jira returns a 200 status code even if the token is invalid.
+ // Jira returns a default dashboard page.
+ if (res.StatusCode >= 200 && res.StatusCode < 300) && res.Header.Get(loginReasonHeaderKey) != failedAuth {
+ s1.Verified = true
+ } else {
+ s1.VerificationError = err
+ }
+ }
+ }
+
+ if !s1.Verified {
+ if detectors.IsKnownFalsePositive(string(s1.Raw), detectors.DefaultFalsePositives, true) {
+ continue
+ }
+ }
+
+ results = append(results, s1)
+
+ }
+
+ }
+ }
+
+ return results, nil
+}
+
+func (s Scanner) Type() detectorspb.DetectorType {
+ return detectorspb.DetectorType_JiraToken
+}
diff --git a/pkg/detectors/jiratoken_v2/jiratoken_v2_test.go b/pkg/detectors/jiratoken_v2/jiratoken_v2_test.go
new file mode 100644
index 000000000..cdcc6e875
--- /dev/null
+++ b/pkg/detectors/jiratoken_v2/jiratoken_v2_test.go
@@ -0,0 +1,123 @@
+//go:build detectors
+// +build detectors
+
+package jiratoken_v2
+
+import (
+ "context"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestJiraToken_FromChunk(t *testing.T) {
+ ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
+ defer cancel()
+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")
+ if err != nil {
+ t.Fatalf("could not get test secrets from GCP: %s", err)
+ }
+ token := testSecrets.MustGetField("JIRA_V2_TOKEN")
+ inactiveToken := testSecrets.MustGetField("JIRA_V2_INACTIVE")
+ email := testSecrets.MustGetField("JIRA_V2_EMAIL")
+ domain := testSecrets.MustGetField("JIRA_V2_DOMAIN")
+
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ s Scanner
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "found, verified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s with jira %s", token, email, domain)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_JiraToken,
+ Verified: true,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found, unverified",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a jira secret %s within jira %s but not jira %s valid", inactiveToken, email, domain)), // the secret would satisfy the regex but not pass validation
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_JiraToken,
+ Verified: false,
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "not found",
+ s: Scanner{},
+ args: args{
+ ctx: context.Background(),
+ data: []byte("You cannot find the secret within"),
+ verify: true,
+ },
+ want: nil,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if (err != nil) != tt.wantErr {
+ t.Errorf("JiraToken.FromData() error = %v, wantErr %v", err, tt.wantErr)
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatalf("no raw secret present: \n %+v", got[i])
+ }
+ got[i].Raw = nil
+ got[i].RawV2 = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
+
+func BenchmarkFromData(benchmark *testing.B) {
+ ctx := context.Background()
+ s := Scanner{}
+ for name, data := range detectors.MustGetBenchmarkData() {
+ benchmark.Run(name, func(b *testing.B) {
+ b.ResetTimer()
+ for n := 0; n < b.N; n++ {
+ _, err := s.FromData(ctx, false, data)
+ if err != nil {
+ b.Fatal(err)
+ }
+ }
+ })
+ }
+}