Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Identify transient AWS verification failures (#1563)
Cody Rose61bee6c8b100
diff --git a/pkg/common/http.go b/pkg/common/http.goindex 56e587fde..1929ca927 100644--- a/pkg/common/http.go+++ b/pkg/common/http.go@@ -100,14 +100,14 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {return &CustomTransport{T}}-func ConstantStatusHttpClient(statusCode int) *http.Client {+func ConstantResponseHttpClient(statusCode int, body string) *http.Client {return &http.Client{Timeout: DefaultResponseTimeout,Transport: FakeTransport{CreateResponse: func(req *http.Request) (*http.Response, error) {return &http.Response{Request: req,- Body: io.NopCloser(strings.NewReader("")),+ Body: io.NopCloser(strings.NewReader(body)),StatusCode: statusCode,}, nil},diff --git a/pkg/detectors/alchemy/alchemy_test.go b/pkg/detectors/alchemy/alchemy_test.goindex ed2ae7b8d..0e14eff65 100644--- a/pkg/detectors/alchemy/alchemy_test.go+++ b/pkg/detectors/alchemy/alchemy_test.go@@ -104,7 +104,7 @@ func TestAlchemy_FromChunk(t *testing.T) {},{name: "found, verified but unexpected api surface",- s: Scanner{client: common.ConstantStatusHttpClient(404)},+ s: Scanner{client: common.ConstantResponseHttpClient(404, "")},args: args{ctx: context.Background(),data: []byte(fmt.Sprintf("You can find a alchemy secret %s within", secret)),diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.goindex b9f96c061..a0177de13 100644--- a/pkg/detectors/aws/aws.go+++ b/pkg/detectors/aws/aws.go@@ -18,7 +18,8 @@ import ()type scanner struct {- skipIDs map[string]struct{}+ verificationClient *http.Client+ skipIDs map[string]struct{}}func New(opts ...func(*scanner)) *scanner {@@ -48,7 +49,7 @@ func WithSkipIDs(skipIDs []string) func(*scanner) {var _ detectors.Detector = (*scanner)(nil)var (- client = common.SaneHttpClient()+ defaultVerificationClient = common.SaneHttpClient()// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.// Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids@@ -123,8 +124,9 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (resulthost := "sts.amazonaws.com"region := "us-east-1"endpoint := "https://sts.amazonaws.com"- datestamp := time.Now().UTC().Format("20060102")- amzDate := time.Now().UTC().Format("20060102T150405Z0700")+ now := time.Now().UTC()+ datestamp := now.Format("20060102")+ amzDate := now.Format("20060102T150405Z0700")req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)if err != nil {@@ -171,6 +173,10 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreq.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")req.URL.RawQuery = params.Encode()+ client := s.verificationClient+ if client == nil {+ client = defaultVerificationClient+ }res, err := client.Do(req)if err == nil {@@ -194,7 +200,25 @@ func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (resultcontinue}- if res.StatusCode != 403 {+ if res.StatusCode == 403 {+ var body awsErrorResponseBody+ err = json.NewDecoder(res.Body).Decode(&body)+ res.Body.Close()+ if err == nil {+ // All instances of the code I've seen in the wild are PascalCased but this check is+ // case-insensitive out of an abundance of caution+ if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {+ // determinate failure - nothing to do+ } else {+ // We see a surprising number of false-negative signature mismatch errors here+ // (The official SDK somehow elicits even more than just making the request+ // ourselves)+ s1.VerificationError = fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)+ }+ } else {+ s1.VerificationError = fmt.Errorf("couldn't parse the sts response body (%v)", err)+ }+ } else {s1.VerificationError = fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)}}@@ -245,6 +269,15 @@ func awsCustomCleanResults(results []detectors.Result) []detectors.Result {return out}+type awsError struct {+ Code string `json:"Code"`+ Message string `json:"Message"`+}++type awsErrorResponseBody struct {+ Error awsError `json:"Error"`+}+type identityRes struct {GetCallerIdentityResponse struct {GetCallerIdentityResult struct {diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/aws_test.goindex 1dfecccd3..582c1ff3c 100644--- a/pkg/detectors/aws/aws_test.go+++ b/pkg/detectors/aws/aws_test.go@@ -17,6 +17,8 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")+var unverifiedSecretClient = common.ConstantResponseHttpClient(403, `{"Error": {"Code": "InvalidClientTokenId"} }`)+func TestAWS_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()@@ -69,7 +71,7 @@ func TestAWS_FromChunk(t *testing.T) {},{name: "found, unverified",- s: scanner{},+ s: scanner{verificationClient: unverifiedSecretClient},args: args{ctx: context.Background(),data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", inactiveSecret, id)), // the secret would satisfy the regex but not pass validation@@ -163,7 +165,7 @@ func TestAWS_FromChunk(t *testing.T) {},{name: "found, unverified, with leading +",- s: scanner{},+ s: scanner{verificationClient: unverifiedSecretClient},args: args{ctx: context.Background(),data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s but not valid", "+HaNv9cTwheDKGJaws/+BMF2GgybQgBWdhcOOdfF", id)), // the secret would satisfy the regex but not pass validation@@ -194,9 +196,45 @@ func TestAWS_FromChunk(t *testing.T) {},{name: "found, would be verified if not for http timeout",- s: scanner{},+ s: scanner{verificationClient: common.SaneHttpClientTimeOut(1 * time.Microsecond)},args: args{- ctx: timeoutContext(1 * time.Microsecond),+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: "AKIASP2TPHJSQH3FJRUX",+ },+ },+ wantErr: false,+ wantVerificationError: true,+ },+ {+ name: "found, unverified due to unexpected http response status",+ s: scanner{verificationClient: common.ConstantResponseHttpClient(500, "internal server error")},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_AWS,+ Verified: false,+ Redacted: "AKIASP2TPHJSQH3FJRUX",+ },+ },+ wantErr: false,+ wantVerificationError: true,+ },+ {+ name: "found, unverified due to unexpected 403 response reason",+ s: scanner{verificationClient: common.ConstantResponseHttpClient(403, `{"Error": {"Code": "SignatureDoesNotMatch"} }`)},+ args: args{+ ctx: context.Background(),data: []byte(fmt.Sprintf("You can find a aws secret %s within aws %s", secret, id)),verify: true,},@@ -224,7 +262,7 @@ func TestAWS_FromChunk(t *testing.T) {t.Fatalf("no raw secret present: \n %+v", got[i])}if (got[i].VerificationError != nil) != tt.wantVerificationError {- t.Fatalf("verification error = %v, wantVerificationError %v", got[i].VerificationError, tt.wantVerificationError)+ t.Fatalf("wantVerificationError %v, verification error = %v", tt.wantVerificationError, got[i].VerificationError)}}ignoreOpts := cmpopts.IgnoreFields(detectors.Result{}, "RawV2", "Raw", "VerificationError")@@ -249,8 +287,3 @@ func BenchmarkFromData(benchmark *testing.B) {})}}--func timeoutContext(timeout time.Duration) context.Context {- c, _ := context.WithTimeout(context.Background(), timeout)- return c-}