Skip to content

Find, verify, and analyze leaked credentials

Identify transient AWS verification failures (#1563)

Cody Rose61bee6c8b100
diff --git a/pkg/common/http.go b/pkg/common/http.go
index 56e587fde..1929ca927 100644
--- a/pkg/common/http.go
+++ b/pkg/common/http.go
@@ -100,14 +100,14 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {
return &CustomTransport{T}
}
-func ConstantStatusHttpClient(statusCode int) *http.Client {
+func ConstantResponseHttpClient(statusCode int, body string) *http.Client {
return &http.Client{
Timeout: DefaultResponseTimeout,
Transport: FakeTransport{
CreateResponse: func(req *http.Request) (*http.Response, error) {
return &http.Response{
Request: req,
- Body: io.NopCloser(strings.NewReader("")),
+ Body: io.NopCloser(strings.NewReader(body)),
StatusCode: statusCode,
}, nil
},
diff --git a/pkg/detectors/alchemy/alchemy_test.go b/pkg/detectors/alchemy/alchemy_test.go
index ed2ae7b8d..0e14eff65 100644
--- a/pkg/detectors/alchemy/alchemy_test.go
+++ b/pkg/detectors/alchemy/alchemy_test.go
@@ -104,7 +104,7 @@ func TestAlchemy_FromChunk(t *testing.T) {
},
{
name: "found, verified but unexpected api surface",
- s: Scanner{client: common.ConstantStatusHttpClient(404)},
+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),
diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.go
index b9f96c061..a0177de13 100644
--- a/pkg/detectors/aws/aws.go
+++ b/pkg/detectors/aws/aws.go
@@ -18,7 +18,8 @@ import (
)
type scanner struct {
- skipIDs map[string]struct{}
+ verificationClient *http.Client
+ skipIDs map[string]struct{}
}
func New(opts ...func(*scanner)) *scanner {
@@ -48,7 +49,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {
var _ detectors.Detector = (*scanner)(nil)
var (
- client = common.SaneHttpClient()
+ defaultVerificationClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
@@ -123,8 +124,9 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
host := "sts.amazonaws.com"
region := "us-east-1"
endpoint := "https://sts.amazonaws.com"
- datestamp := time.Now().UTC().Format("20060102")
- amzDate := time.Now().UTC().Format("20060102T150405Z0700")
+ now := time.Now().UTC()
+ datestamp := now.Format("20060102")
+ amzDate := now.Format("20060102T150405Z0700")
req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
if err != nil {
@@ -171,6 +173,10 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
req.URL.RawQuery = params.Encode()
+ client := s.verificationClient
+ if client == nil {
+ client = defaultVerificationClient
+ }
res, err := client.Do(req)
if err == nil {
@@ -194,7 +200,25 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
- if res.StatusCode != 403 {
+ if res.StatusCode == 403 {
+ var body awsErrorResponseBody
+ err = json.NewDecoder(res.Body).Decode(&body)
+ res.Body.Close()
+ if err == nil {
+ // All instances of the code I've seen in the wild are PascalCased but this check is
+ // case-insensitive out of an abundance of caution
+ if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
+ // determinate failure - nothing to do
+ } else {
+ // We see a surprising number of false-negative signature mismatch errors here
+ // (The official SDK somehow elicits even more than just making the request
+ // ourselves)
+ s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
+ }
+ } else {
+ s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)
+ }
+ } else {
s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
}
}
@@ -245,6 +269,15 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {
return out
}
+type awsError struct {
+ Code string `json:"Code"`
+ Message string `json:"Message"`
+}
+
+type awsErrorResponseBody struct {
+ Error awsError `json:"Error"`
+}
+
type identityRes struct {
GetCallerIdentityResponse struct {
GetCallerIdentityResult struct {
diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.go
index 1dfecccd3..582c1ff3c 100644
--- a/pkg/detectors/aws/aws_test.go
+++ b/pkg/detectors/aws/aws_test.go
@@ -17,6 +17,8 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
+var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)
+
func TestAWS_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()
@@ -69,7 +71,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, unverified",
- s: scanner{},
+ s: scanner{verificationClient: unverifiedSecretClient},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation
@@ -163,7 +165,7 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, unverified, with leading +",
- s: scanner{},
+ s: scanner{verificationClient: unverifiedSecretClient},
args: args{
ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation
@@ -194,9 +196,45 @@ func TestAWS_FromChunk(t *testing.T) {
},
{
name: "found, would be verified if not for http timeout",
- s: scanner{},
+ s: scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},
args: args{
- ctx: timeoutContext(1 * time.Microsecond),
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: "AKIASP2TPHJSQH3FJRUX",
+ },
+ },
+ wantErr: false,
+ wantVerificationError: true,
+ },
+ {
+ name: "found, unverified due to unexpected http response status",
+ s: scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_AWS,
+ Verified: false,
+ Redacted: "AKIASP2TPHJSQH3FJRUX",
+ },
+ },
+ wantErr: false,
+ wantVerificationError: true,
+ },
+ {
+ name: "found, unverified due to unexpected 403 response reason",
+ s: scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},
+ args: args{
+ ctx: context.Background(),
data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),
verify: true,
},
@@ -224,7 +262,7 @@ func TestAWS_FromChunk(t *testing.T) {
t.Fatalf("no raw secret present: \n %+v", got[i])
}
if (got[i].VerificationError != nil) != tt.wantVerificationError {
- t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationError)
+ t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError)
}
}
ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "VerificationError")
@@ -249,8 +287,3 @@ func BenchmarkFromData(benchmark *testing.B) {
})
}
}
-
-func timeoutContext(timeout time.Duration) context.Context {
- c, _ := context.WithTimeout(context.Background(), timeout)
- return c
-}