Skip to content

Find, verify, and analyze leaked credentials

Ignore canary IDs in notifications (#2526)

Dylan Ayrey7620906b075a
diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.go
index f46d4fa8d..d7c2b932e 100644
--- a/pkg/detectors/aws/aws.go
+++ b/pkg/detectors/aws/aws.go
@@ -21,6 +21,7 @@ import (
type scanner struct {
verificationClient *http.Client
skipIDs map[string]struct{}
+ verifyCanaries bool
}
// resourceTypes derived from: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
@@ -95,6 +96,12 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
}
}
+func WithVerifyCanaries() func(*scanner) {
+ return func(s *scanner) {
+ s.verifyCanaries = true
+ }
+}
+
// Ensure the scanner satisfies the interface at compile time.
var _ detectors.Detector = (*scanner)(nil)
@@ -173,15 +180,17 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if err == nil {
s1.ExtraData["account"] = account
}
- if _, ok := thinkstCanaryList[account]; ok {
- s1.ExtraData["is_canary"] = "true"
- s1.ExtraData["message"] = thinkstMessage
- s1.Verified = true
- }
- if _, ok := thinkstKnockoffsCanaryList[account]; ok {
- s1.ExtraData["is_canary"] = "true"
- s1.ExtraData["message"] = thinkstKnockoffsMessage
- s1.Verified = true
+ if !s.verifyCanaries && !strings.Contains(dataStr, "Redacted: "+resIDMatch) {
+ if _, ok := thinkstCanaryList[account]; ok {
+ s1.ExtraData["is_canary"] = "true"
+ s1.ExtraData["message"] = thinkstMessage
+ s1.Verified = true
+ }
+ if _, ok := thinkstKnockoffsCanaryList[account]; ok {
+ s1.ExtraData["is_canary"] = "true"
+ s1.ExtraData["message"] = thinkstKnockoffsMessage
+ s1.Verified = true
+ }
}
if verify && !s1.Verified {
diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.go
index 8ab69552e..10ab5c1bc 100644
--- a/pkg/detectors/aws/aws_test.go
+++ b/pkg/detectors/aws/aws_test.go
@@ -49,7 +49,7 @@ func TestAWS_FromChunk(t *testing.T) {
}{
{
name: "found, verified",
- s: scanner{},
+ s: scanner{verifyCanaries: true},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -73,7 +73,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, unverified",
- s: scanner{verificationClient: unverifiedSecretClient},
+ s: scanner{verificationClient: unverifiedSecretClient, verifyCanaries: true},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
@@ -105,7 +105,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found two, one included for every ID found",
- s: scanner{},
+ s: scanner{verifyCanaries: true},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and this is the secret %s", id, secret, inactiveID, inactiveSecret)),
@@ -149,7 +149,9 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found two, returned both because the active secret for one paired with the inactive ID, despite the hash",
- s: scanner{},
+ s: scanner{
+ verifyCanaries: true,
+ },
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("The verified ID is %s with a secret of %s, but the unverified ID is %s and the secret is this hash %s", id, secret, inactiveID, hash)),
@@ -172,13 +174,17 @@ func TestAWS_FromChunk(t *testing.T) {
DetectorType: detectorspb.DetectorType_AWS,
Verified: false,
Redacted: inactiveID,
+ ExtraData: map[string]string{"account": "171436882533", "resource_type": "Access key"},
},
},
wantErr: false,
},
{
name: "found, unverified, with leading +",
- s: scanner{verificationClient: unverifiedSecretClient},
+ s: scanner{
+ verificationClient: unverifiedSecretClient,
+ verifyCanaries: true,
+ },
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
@@ -213,7 +219,10 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, would be verified if not for http timeout",
- s: scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
+ s: scanner{
+ verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond),
+ verifyCanaries: true,
+ },
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -235,7 +244,10 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, unverified due to unexpected http response status",
- s: scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},
+ s: scanner{
+ verificationClient: common.ConstantResponseHttpClient(500, "internal server error"),
+ verifyCanaries: true,
+ },
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -257,7 +269,10 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, unverified due to unexpected 403 response reason",
- s: scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},
+ s: scanner{
+ verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`),
+ verifyCanaries: true,
+ },
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
@@ -279,7 +294,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "verified secret checked directly after unverified secret with same key id",
- s: scanner{},
+ s: scanner{verifyCanaries: true},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("%s\n%s\n%s", inactiveSecret, id, secret)),