Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Ensure multipart credentials are deduplicated correctly (#1271)
Dustin Deckere217e2fbfd77
diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.goindex f289451b8..3628a9f58 100644--- a/pkg/detectors/aws/aws_test.go+++ b/pkg/detectors/aws/aws_test.go@@ -56,11 +56,11 @@ func TestAWS_FromChunk(t *testing.T) {{DetectorType: detectorspb.DetectorType_AWS,Verified: true,- Redacted: "AKIAWARWQKZNHMZBLY4I",+ Redacted: "AKIASP2TPHJSQH3FJRUX",ExtraData: map[string]string{- "account": "413504919130",- "arn": "arn:aws:iam::413504919130:root",- "user_id": "413504919130",+ "account": "171436882533",+ "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",+ "user_id": "AIDASP2TPHJSUFRSTTZX4",},},},@@ -78,7 +78,7 @@ func TestAWS_FromChunk(t *testing.T) {{DetectorType: detectorspb.DetectorType_AWS,Verified: false,- Redacted: "AKIAWARWQKZNHMZBLY4I",+ Redacted: "AKIASP2TPHJSQH3FJRUX",ExtraData: nil,},},@@ -106,18 +106,18 @@ func TestAWS_FromChunk(t *testing.T) {want: []detectors.Result{{DetectorType: detectorspb.DetectorType_AWS,- Verified: true,- Redacted: id,- ExtraData: map[string]string{- "account": "413504919130",- "arn": "arn:aws:iam::413504919130:root",- "user_id": "413504919130",- },+ Verified: false,+ Redacted: "AKIASP2TPHJSQH3FJXYZ",},{DetectorType: detectorspb.DetectorType_AWS,- Verified: false,- Redacted: inactiveID,+ Verified: true,+ Redacted: "AKIASP2TPHJSQH3FJRUX",+ ExtraData: map[string]string{+ "account": "171436882533",+ "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",+ "user_id": "AIDASP2TPHJSUFRSTTZX4",+ },},},wantErr: false,@@ -145,11 +145,11 @@ func TestAWS_FromChunk(t *testing.T) {{DetectorType: detectorspb.DetectorType_AWS,Verified: true,- Redacted: id,+ Redacted: "AKIASP2TPHJSQH3FJRUX",ExtraData: map[string]string{- "account": "413504919130",- "arn": "arn:aws:iam::413504919130:root",- "user_id": "413504919130",+ "account": "171436882533",+ "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",+ "user_id": "AIDASP2TPHJSUFRSTTZX4",},},{@@ -172,7 +172,7 @@ func TestAWS_FromChunk(t *testing.T) {{DetectorType: detectorspb.DetectorType_AWS,Verified: false,- Redacted: "AKIAWARWQKZNHMZBLY4I",+ Redacted: "AKIASP2TPHJSQH3FJRUX",},},wantErr: false,@@ -181,7 +181,7 @@ func TestAWS_FromChunk(t *testing.T) {name: "skipped",s: scanner{skipIDs: map[string]struct{}{- "AKIAWARWQKZNHMZBLY4I": {},+ "AKIASP2TPHJSQH3FJRUX": {},},},args: args{diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.goindex b026f99ea..5ec70a943 100644--- a/pkg/detectors/gcp/gcp.go+++ b/pkg/detectors/gcp/gcp.go@@ -79,14 +79,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultraw = []byte(key)}+ credBytes, _ := json.Marshal(creds)+s := detectors.Result{DetectorType: detectorspb.DetectorType_GCP,Raw: raw,+ RawV2: credBytes,Redacted: creds.ClientEmail,}if verify {- credBytes, _ := json.Marshal(creds)credentials, err := google.CredentialsFromJSON(ctx, credBytes, "https://www.googleapis.com/auth/cloud-platform")if err != nil {continuediff --git a/pkg/detectors/jiratoken/jiratoken.go b/pkg/detectors/jiratoken/jiratoken.goindex a04a58c98..678d28511 100644--- a/pkg/detectors/jiratoken/jiratoken.go+++ b/pkg/detectors/jiratoken/jiratoken.go@@ -60,6 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results1 := detectors.Result{DetectorType: detectorspb.DetectorType_JiraToken,Raw: []byte(resToken),+ RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),}if verify {diff --git a/pkg/detectors/jiratoken/jiratoken_test.go b/pkg/detectors/jiratoken/jiratoken_test.goindex aaddcd03b..2c0486124 100644--- a/pkg/detectors/jiratoken/jiratoken_test.go+++ b/pkg/detectors/jiratoken/jiratoken_test.go@@ -97,6 +97,7 @@ func TestJiraToken_FromChunk(t *testing.T) {t.Fatalf("no raw secret present: \n %+v", got[i])}got[i].Raw = nil+ got[i].RawV2 = nil}if diff := pretty.Compare(got, tt.want); diff != "" {t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.goindex 5c85e384b..ecaef8a27 100644--- a/pkg/detectors/okta/okta.go+++ b/pkg/detectors/okta/okta.go@@ -41,6 +41,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results := detectors.Result{DetectorType: detectorspb.DetectorType_Okta,Raw: []byte(token),+ RawV2: []byte(fmt.Sprintf("%s:%s", domain, token)),}if verify {diff --git a/pkg/detectors/onelogin/onelogin.go b/pkg/detectors/onelogin/onelogin.goindex 0ef614d13..0212a0681 100644--- a/pkg/detectors/onelogin/onelogin.go+++ b/pkg/detectors/onelogin/onelogin.go@@ -50,6 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results := detectors.Result{DetectorType: detectorspb.DetectorType_OneLogin,Raw: []byte(clientID[1]),+ RawV2: []byte(fmt.Sprintf("%s:%s", clientID[1], clientSecret[1])),Redacted: clientID[1],}diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.goindex 203b048ae..05d00f7ce 100644--- a/pkg/detectors/uri/uri.go+++ b/pkg/detectors/uri/uri.go@@ -66,12 +66,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}rawURL, _ := url.Parse(urlMatch)+ rawURLStr := rawURL.String()+ // Removing the path causes possible deduplication issues if some paths have basic auth and some do not.rawURL.Path = ""redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))s := detectors.Result{DetectorType: detectorspb.DetectorType_URI,Raw: []byte(rawURL.String()),+ RawV2: []byte(rawURLStr),Redacted: redact,}diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.goindex 1cd024e3f..ba090c0e2 100644--- a/pkg/detectors/uri/uri_test.go+++ b/pkg/detectors/uri/uri_test.go@@ -38,7 +38,7 @@ func TestURI_FromChunk(t *testing.T) {{DetectorType: detectorspb.DetectorType_URI,Verified: false,- Redacted: "https://user:****@www.httpwatch.com",+ Redacted: "https://user:********@www.httpwatch.com",},},wantErr: false,@@ -55,7 +55,7 @@ func TestURI_FromChunk(t *testing.T) {{DetectorType: detectorspb.DetectorType_URI,Verified: true,- Redacted: "https://httpwatch:****@www.httpwatch.com",+ Redacted: "https://httpwatch:********@www.httpwatch.com",},},wantErr: false,@@ -72,7 +72,7 @@ func TestURI_FromChunk(t *testing.T) {{DetectorType: detectorspb.DetectorType_URI,Verified: true,- Redacted: "https://httpwatch:****@www.httpwatch.com",+ Redacted: "https://httpwatch:********@www.httpwatch.com",},},wantErr: false,@@ -101,6 +101,7 @@ func TestURI_FromChunk(t *testing.T) {// }for i := range got {got[i].Raw = nil+ got[i].RawV2 = nil}if diff := pretty.Compare(got, tt.want); diff != "" {t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)