Skip to content

Find, verify, and analyze leaked credentials

Ensure multipart credentials are deduplicated correctly (#1271)

Dustin Deckere217e2fbfd77
diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.go
index f289451b8..3628a9f58 100644
--- a/pkg/detectors/aws/aws_test.go
+++ b/pkg/detectors/aws/aws_test.go
@@ -56,11 +56,11 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
- Redacted: "AKIAWARWQKZNHMZBLY4I",
+ Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
- "account": "413504919130",
- "arn": "arn:aws:iam::413504919130:root",
- "user_id": "413504919130",
+ "account": "171436882533",
+ "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
+ "user_id": "AIDASP2TPHJSUFRSTTZX4",
},
},
},
@@ -78,7 +78,7 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
- Redacted: "AKIAWARWQKZNHMZBLY4I",
+ Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: nil,
},
},
@@ -106,18 +106,18 @@ func TestAWS_FromChunk(t *testing.T) {
want: []detectors.Result{
{
DetectorType: detectorspb.DetectorType_AWS,
- Verified: true,
- Redacted: id,
- ExtraData: map[string]string{
- "account": "413504919130",
- "arn": "arn:aws:iam::413504919130:root",
- "user_id": "413504919130",
- },
+ Verified: false,
+ Redacted: "AKIASP2TPHJSQH3FJXYZ",
},
{
DetectorType: detectorspb.DetectorType_AWS,
- Verified: false,
- Redacted: inactiveID,
+ Verified: true,
+ Redacted: "AKIASP2TPHJSQH3FJRUX",
+ ExtraData: map[string]string{
+ "account": "171436882533",
+ "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
+ "user_id": "AIDASP2TPHJSUFRSTTZX4",
+ },
},
},
wantErr: false,
@@ -145,11 +145,11 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: true,
- Redacted: id,
+ Redacted: "AKIASP2TPHJSQH3FJRUX",
ExtraData: map[string]string{
- "account": "413504919130",
- "arn": "arn:aws:iam::413504919130:root",
- "user_id": "413504919130",
+ "account": "171436882533",
+ "arn": "arn:aws:iam::171436882533:user/canarytokens.com@@4dxkh0pdeop3bzu9zx5wob793",
+ "user_id": "AIDASP2TPHJSUFRSTTZX4",
},
},
{
@@ -172,7 +172,7 @@ func TestAWS_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
- Redacted: "AKIAWARWQKZNHMZBLY4I",
+ Redacted: "AKIASP2TPHJSQH3FJRUX",
},
},
wantErr: false,
@@ -181,7 +181,7 @@ func TestAWS_FromChunk(t *testing.T) {
name: "skipped",
s: scanner{
skipIDs: map[string]struct{}{
- "AKIAWARWQKZNHMZBLY4I": {},
+ "AKIASP2TPHJSQH3FJRUX": {},
},
},
args: args{
diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.go
index b026f99ea..5ec70a943 100644
--- a/pkg/detectors/gcp/gcp.go
+++ b/pkg/detectors/gcp/gcp.go
@@ -79,14 +79,16 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
raw = []byte(key)
}
+ credBytes, _ := json.Marshal(creds)
+
s := detectors.Result{
DetectorType: detectorspb.DetectorType_GCP,
Raw: raw,
+ RawV2: credBytes,
Redacted: creds.ClientEmail,
}
if verify {
- credBytes, _ := json.Marshal(creds)
credentials, err := google.CredentialsFromJSON(ctx, credBytes, "https://www.googleapis.com/auth/cloud-platform")
if err != nil {
continue
diff --git a/pkg/detectors/jiratoken/jiratoken.go b/pkg/detectors/jiratoken/jiratoken.go
index a04a58c98..678d28511 100644
--- a/pkg/detectors/jiratoken/jiratoken.go
+++ b/pkg/detectors/jiratoken/jiratoken.go
@@ -60,6 +60,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_JiraToken,
Raw: []byte(resToken),
+ RawV2: []byte(fmt.Sprintf("%s:%s:%s", resEmail, resToken, resDomain)),
}
if verify {
diff --git a/pkg/detectors/jiratoken/jiratoken_test.go b/pkg/detectors/jiratoken/jiratoken_test.go
index aaddcd03b..2c0486124 100644
--- a/pkg/detectors/jiratoken/jiratoken_test.go
+++ b/pkg/detectors/jiratoken/jiratoken_test.go
@@ -97,6 +97,7 @@ func TestJiraToken_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
got[i].Raw = nil
+ got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("JiraToken.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.go
index 5c85e384b..ecaef8a27 100644
--- a/pkg/detectors/okta/okta.go
+++ b/pkg/detectors/okta/okta.go
@@ -41,6 +41,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s := detectors.Result{
DetectorType: detectorspb.DetectorType_Okta,
Raw: []byte(token),
+ RawV2: []byte(fmt.Sprintf("%s:%s", domain, token)),
}
if verify {
diff --git a/pkg/detectors/onelogin/onelogin.go b/pkg/detectors/onelogin/onelogin.go
index 0ef614d13..0212a0681 100644
--- a/pkg/detectors/onelogin/onelogin.go
+++ b/pkg/detectors/onelogin/onelogin.go
@@ -50,6 +50,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
s := detectors.Result{
DetectorType: detectorspb.DetectorType_OneLogin,
Raw: []byte(clientID[1]),
+ RawV2: []byte(fmt.Sprintf("%s:%s", clientID[1], clientSecret[1])),
Redacted: clientID[1],
}
diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go
index 203b048ae..05d00f7ce 100644
--- a/pkg/detectors/uri/uri.go
+++ b/pkg/detectors/uri/uri.go
@@ -66,12 +66,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
rawURL, _ := url.Parse(urlMatch)
+ rawURLStr := rawURL.String()
+ // Removing the path causes possible deduplication issues if some paths have basic auth and some do not.
rawURL.Path = ""
redact := strings.TrimSpace(strings.Replace(rawURL.String(), password, "********", -1))
s := detectors.Result{
DetectorType: detectorspb.DetectorType_URI,
Raw: []byte(rawURL.String()),
+ RawV2: []byte(rawURLStr),
Redacted: redact,
}
diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.go
index 1cd024e3f..ba090c0e2 100644
--- a/pkg/detectors/uri/uri_test.go
+++ b/pkg/detectors/uri/uri_test.go
@@ -38,7 +38,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: false,
- Redacted: "https://user:****@www.httpwatch.com",
+ Redacted: "https://user:********@www.httpwatch.com",
},
},
wantErr: false,
@@ -55,7 +55,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: true,
- Redacted: "https://httpwatch:****@www.httpwatch.com",
+ Redacted: "https://httpwatch:********@www.httpwatch.com",
},
},
wantErr: false,
@@ -72,7 +72,7 @@ func TestURI_FromChunk(t *testing.T) {
{
DetectorType: detectorspb.DetectorType_URI,
Verified: true,
- Redacted: "https://httpwatch:****@www.httpwatch.com",
+ Redacted: "https://httpwatch:********@www.httpwatch.com",
},
},
wantErr: false,
@@ -101,6 +101,7 @@ func TestURI_FromChunk(t *testing.T) {
// }
for i := range got {
got[i].Raw = nil
+ got[i].RawV2 = nil
}
if diff := pretty.Compare(got, tt.want); diff != "" {
t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)