Skip to content

Find, verify, and analyze leaked credentials

feat: cleanup AWS detector logic (#3583)

Richard Gomezd408849c977d
diff --git a/pkg/common/utils.go b/pkg/common/utils.go
index a8f38ffd7..ea8d5e4e7 100644
--- a/pkg/common/utils.go
+++ b/pkg/common/utils.go
@@ -3,9 +3,6 @@ package common
import (
"bufio"
"crypto/rand"
- "encoding/base32"
- "encoding/binary"
- "fmt"
"io"
"math/big"
"strings"
@@ -55,33 +52,6 @@ func RandomID(length int) string {
return string(b)
}
-func GetAccountNumFromAWSID(AWSID string) (string, error) {
- // Function to get the account number from an AWS ID (no verification required)
- // Source: https://medium.com/@TalBeerySec/a-short-note-on-aws-key-id-f88cc4317489
- if len(AWSID) < 4 {
- return "", fmt.Errorf("AWSID is too short")
- }
- if AWSID[4] == 'I' || AWSID[4] == 'J' {
- return "", fmt.Errorf("Can't get account number from AKIAJ/ASIAJ or AKIAI/ASIAI keys")
- }
- trimmedAWSID := AWSID[4:]
- decodedBytes, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(trimmedAWSID))
- if err != nil {
- return "", err
- }
-
- if len(decodedBytes) < 6 {
- return "", fmt.Errorf("Decoded AWSID is too short")
- }
-
- data := make([]byte, 8)
- copy(data[2:], decodedBytes[0:6])
- z := binary.BigEndian.Uint64(data)
- const mask uint64 = 0x7fffffffff80
- accountNum := (z & mask) >> 7
- return fmt.Sprintf("%012d", accountNum), nil
-}
-
// SliceContainsString searches a slice to determine if it contains a specified string.
// Returns the index of the first match in the slice.
func SliceContainsString(origTargetString string, stringSlice []string, ignoreCase bool) (bool, string, int) {
diff --git a/pkg/detectors/aws/access_keys/accesskey.go b/pkg/detectors/aws/access_keys/accesskey.go
new file mode 100644
index 000000000..e00776d98
--- /dev/null
+++ b/pkg/detectors/aws/access_keys/accesskey.go
@@ -0,0 +1,325 @@
+package access_keys
+
+import (
+ "context"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ regexp "github.com/wasilibs/go-re2"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type scanner struct {
+ verificationClient *http.Client
+ skipIDs map[string]struct{}
+ detectors.DefaultMultiPartCredentialProvider
+}
+
+func New(opts ...func(*scanner)) *scanner {
+ scanner := &scanner{
+ skipIDs: map[string]struct{}{},
+ }
+ for _, opt := range opts {
+ opt(scanner)
+ }
+
+ return scanner
+}
+
+func WithSkipIDs(skipIDs []string) func(*scanner) {
+ return func(s *scanner) {
+ ids := map[string]struct{}{}
+ for _, id := range skipIDs {
+ ids[id] = struct{}{}
+ }
+
+ s.skipIDs = ids
+ }
+}
+
+// Ensure the scanner satisfies the interface at compile time.
+var _ interface {
+ detectors.Detector
+ detectors.CustomResultsCleaner
+ detectors.MultiPartCredentialProvider
+} = (*scanner)(nil)
+
+var (
+ defaultVerificationClient = common.SaneHttpClient()
+
+ // Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
+ // Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
+ idPat = regexp.MustCompile(`\b((?:AKIA|ABIA|ACCA)[A-Z0-9]{16})\b`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s scanner) Keywords() []string {
+ return []string{
+ "AKIA",
+ "ABIA",
+ "ACCA",
+ }
+}
+
+// FromData will find and optionally verify AWS secrets in a given set of bytes.
+func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ logger := logContext.AddLogger(ctx).Logger().WithName("aws")
+ dataStr := string(data)
+ dataStr = aws.UrlEncodedReplacer.Replace(dataStr)
+
+ // Filter & deduplicate matches.
+ idMatches := make(map[string]struct{})
+ for _, matches := range idPat.FindAllStringSubmatch(dataStr, -1) {
+ idMatches[matches[1]] = struct{}{}
+ }
+ secretMatches := make(map[string]struct{})
+ for _, matches := range aws.SecretPat.FindAllStringSubmatch(dataStr, -1) {
+ secretMatches[matches[1]] = struct{}{}
+ }
+
+ // Process matches.
+ for idMatch := range idMatches {
+ if detectors.StringShannonEntropy(idMatch) < aws.RequiredIdEntropy {
+ continue
+ }
+ if s.skipIDs != nil {
+ if _, ok := s.skipIDs[idMatch]; ok {
+ continue
+ }
+ }
+
+ for secretMatch := range secretMatches {
+ if detectors.StringShannonEntropy(secretMatch) < aws.RequiredSecretEntropy {
+ continue
+ }
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_AWS,
+ Raw: []byte(idMatch),
+ Redacted: idMatch,
+ RawV2: []byte(idMatch + ":" + secretMatch),
+ ExtraData: map[string]string{
+ "resource_type": aws.ResourceTypes[idMatch[:4]],
+ },
+ }
+
+ // Decode the account ID.
+ account, err := aws.GetAccountNumFromID(idMatch)
+ isCanary := false
+ if err != nil {
+ logger.V(3).Info("Failed to decode account number", "err", err)
+ } else {
+ s1.ExtraData["account"] = account
+
+ // Handle canary IDs.
+ if _, ok := thinkstCanaryList[account]; ok {
+ isCanary = true
+ s1.ExtraData["message"] = thinkstMessage
+ if verify {
+ verified, arn, err := s.verifyCanary(idMatch, secretMatch)
+ s1.Verified = verified
+ if arn != "" {
+ s1.ExtraData["arn"] = arn
+ }
+ s1.SetVerificationError(err, secretMatch)
+ }
+ }
+ if _, ok := thinkstKnockoffsCanaryList[account]; ok {
+ isCanary = true
+ s1.ExtraData["message"] = thinkstKnockoffsMessage
+ if verify {
+ verified, arn, err := s.verifyCanary(idMatch, secretMatch)
+ s1.Verified = verified
+ if arn != "" {
+ s1.ExtraData["arn"] = arn
+ }
+ s1.SetVerificationError(err, secretMatch)
+ }
+ }
+
+ if isCanary {
+ s1.ExtraData["is_canary"] = "true"
+ }
+ }
+
+ if verify && !isCanary {
+ isVerified, extraData, verificationErr := s.verifyMatch(ctx, idMatch, secretMatch, true)
+ s1.Verified = isVerified
+
+ // Log if the calculated ID does not match the ID value from verification.
+ // Should only be edge cases at most.
+ if account != "" && extraData["account"] != "" && extraData["account"] != s1.ExtraData["account"] {
+ logger.V(2).Info("Calculated account ID does not match actual account ID", "calculated", account, "actual", extraData["account"])
+ }
+
+ // Append the extraData to the existing ExtraData map.
+ for k, v := range extraData {
+ s1.ExtraData[k] = v
+ }
+ s1.SetVerificationError(verificationErr, secretMatch)
+ }
+
+ if !s1.Verified && aws.FalsePositiveSecretPat.MatchString(secretMatch) {
+ // Unverified results that look like hashes are probably not secrets
+ continue
+ }
+
+ results = append(results, s1)
+ // If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
+ if s1.Verified {
+ delete(secretMatches, secretMatch)
+ break
+ }
+ }
+ }
+ return results, nil
+}
+
+func (s scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool {
+ return true
+}
+
+const (
+ method = "GET"
+ service = "sts"
+ host = "sts.amazonaws.com"
+ region = "us-east-1"
+ endpoint = "https://sts.amazonaws.com"
+)
+
+func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
+ // REQUEST VALUES.
+ now := time.Now().UTC()
+ datestamp := now.Format("20060102")
+ amzDate := now.Format("20060102T150405Z0700")
+
+ req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
+ if err != nil {
+ return false, nil, err
+ }
+ req.Header.Set("Accept", "application/json")
+
+ // TASK 1: CREATE A CANONICAL REQUEST.
+ // http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+ canonicalURI := "/"
+ canonicalHeaders := "host:" + host + "\n"
+ signedHeaders := "host"
+ algorithm := "AWS4-HMAC-SHA256"
+ credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
+
+ params := req.URL.Query()
+ params.Add("Action", "GetCallerIdentity")
+ params.Add("Version", "2011-06-15")
+ params.Add("X-Amz-Algorithm", algorithm)
+ params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
+ params.Add("X-Amz-Date", amzDate)
+ params.Add("X-Amz-Expires", "30")
+ params.Add("X-Amz-SignedHeaders", signedHeaders)
+
+ canonicalQuerystring := params.Encode()
+ payloadHash := aws.GetHash("") // empty payload
+ canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
+
+ // TASK 2: CREATE THE STRING TO SIGN.
+ stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + aws.GetHash(canonicalRequest)
+
+ // TASK 3: CALCULATE THE SIGNATURE.
+ // https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+ hash := aws.GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
+ hash = aws.GetHMAC(hash, []byte(region))
+ hash = aws.GetHMAC(hash, []byte(service))
+ hash = aws.GetHMAC(hash, []byte("aws4_request"))
+
+ signature2 := aws.GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
+ signature := hex.EncodeToString(signature2)
+
+ // TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
+ params.Add("X-Amz-Signature", signature)
+ req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
+ req.URL.RawQuery = params.Encode()
+
+ client := s.verificationClient
+ if client == nil {
+ client = defaultVerificationClient
+ }
+
+ res, err := client.Do(req)
+ if err != nil {
+ return false, nil, err
+ }
+ defer func() {
+ _, _ = io.Copy(io.Discard, res.Body)
+ _ = res.Body.Close()
+ }()
+
+ // TODO: tighten range of acceptable status codes
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ identityInfo := aws.IdentityResponse{}
+ if err := json.NewDecoder(res.Body).Decode(&identityInfo); err != nil {
+ return false, nil, err
+ }
+
+ extraData := map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+ "account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
+ "user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
+ "arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
+ }
+ return true, extraData, nil
+ } else if res.StatusCode == 403 {
+ // Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
+ // share a key ID but are signed with different secrets the second one will be rejected with a 403 that
+ // carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
+ // valid. Since this is exactly our access pattern, we need to work around it.
+ //
+ // Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
+ // response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
+ // the response to the original second request, or read to its end, or the resubmission will also yield a
+ // SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
+ // SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
+ // after decoding its response body, which requires reading the entire response body, which disables the
+ // workaround.
+ //
+ // We are clearly deep in the guts of AWS implementation details here, so this all might change with no
+ // notice. If you're here because something in this detector broke, you have my condolences.
+ if retryOn403 {
+ return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
+ }
+
+ var body aws.ErrorResponseBody
+ if err = json.NewDecoder(res.Body).Decode(&body); err != nil {
+ return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
+ }
+ // All instances of the code I've seen in the wild are PascalCased but this check is
+ // case-insensitive out of an abundance of caution
+ if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
+ return false, nil, nil
+ }
+ return false, nil, fmt.Errorf("request returned status %d with an unexpected reason (%s: %s)", res.StatusCode, body.Error.Code, body.Error.Message)
+ } else {
+ return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+ }
+}
+
+func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {
+ return aws.CleanResults(results)
+}
+
+func (s scanner) Type() detectorspb.DetectorType {
+ return detectorspb.DetectorType_AWS
+}
+
+func (s scanner) Description() string {
+ return "AWS (Amazon Web Services) is a comprehensive cloud computing platform offering a wide range of on-demand services like computing power, storage, databases. API keys for AWS can have varying amount of access to these services depending on the IAM policy attached."
+}
diff --git a/pkg/detectors/aws/aws_integration_test.go b/pkg/detectors/aws/access_keys/accesskey_integration_test.go
similarity index 99%
rename from pkg/detectors/aws/aws_integration_test.go
rename to pkg/detectors/aws/access_keys/accesskey_integration_test.go
index 92ffa5c3c..a2acaf5a2 100644
--- a/pkg/detectors/aws/aws_integration_test.go
+++ b/pkg/detectors/aws/access_keys/accesskey_integration_test.go
@@ -1,7 +1,7 @@
//go:build detectors
// +build detectors
-package aws
+package access_keys
import (
"context"
diff --git a/pkg/detectors/aws/aws_test.go b/pkg/detectors/aws/access_keys/accesskey_test.go
similarity index 99%
rename from pkg/detectors/aws/aws_test.go
rename to pkg/detectors/aws/access_keys/accesskey_test.go
index 576451fed..227695856 100644
--- a/pkg/detectors/aws/aws_test.go
+++ b/pkg/detectors/aws/access_keys/accesskey_test.go
@@ -1,4 +1,4 @@
-package aws
+package access_keys
import (
"context"
diff --git a/pkg/detectors/aws/access_keys/canary.go b/pkg/detectors/aws/access_keys/canary.go
new file mode 100644
index 000000000..8081bf3f0
--- /dev/null
+++ b/pkg/detectors/aws/access_keys/canary.go
@@ -0,0 +1,76 @@
+package access_keys
+
+import (
+ "strings"
+
+ "github.com/aws/aws-sdk-go/aws"
+ "github.com/aws/aws-sdk-go/aws/credentials"
+ "github.com/aws/aws-sdk-go/aws/session"
+ "github.com/aws/aws-sdk-go/service/sns"
+)
+
+const thinkstMessage = "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries"
+const thinkstKnockoffsMessage = "This is an off brand AWS Canary inspired by canarytokens.org. It wasn't set off; learn more here: https://trufflesecurity.com/canaries"
+
+var (
+ thinkstCanaryList = map[string]struct{}{
+ "052310077262": {},
+ "171436882533": {},
+ "534261010715": {},
+ "595918472158": {},
+ "717712589309": {},
+ "819147034852": {},
+ "992382622183": {},
+ "730335385048": {},
+ }
+ thinkstKnockoffsCanaryList = map[string]struct{}{
+ "044858866125": {},
+ "251535659677": {},
+ "344043088457": {},
+ "351906852752": {},
+ "390477818340": {},
+ "426127672474": {},
+ "427150556519": {},
+ "439872796651": {},
+ "445142720921": {},
+ "465867158099": {},
+ "637958123769": {},
+ "693412236332": {},
+ "732624840810": {},
+ "735421457923": {},
+ "959235150393": {},
+ "982842642351": {},
+ }
+)
+
+func (s scanner) verifyCanary(resIDMatch, resSecretMatch string) (bool, string, error) {
+ // Prep AWS Creds for SNS
+ sess := session.Must(session.NewSession(&aws.Config{
+ Region: aws.String("us-east-1"), // any region seems to work
+ Credentials: credentials.NewStaticCredentials(
+ resIDMatch,
+ resSecretMatch,
+ "",
+ ),
+ HTTPClient: s.verificationClient,
+ }))
+ svc := sns.New(sess)
+
+ // Prep vars and Publish to SNS
+ _, err := svc.Publish(&sns.PublishInput{
+ Message: aws.String("foo"),
+ PhoneNumber: aws.String("1"),
+ })
+
+ if strings.Contains(err.Error(), "not authorized to perform") {
+ arn := strings.Split(err.Error(), "User: ")[1]
+ arn = strings.Split(arn, " is not authorized to perform: ")[0]
+ return true, arn, nil
+ } else if strings.Contains(err.Error(), "does not match the signature you provided") {
+ return false, "", nil
+ } else if strings.Contains(err.Error(), "status code: 403") {
+ return false, "", nil
+ } else {
+ return false, "", err
+ }
+}
diff --git a/pkg/detectors/aws/aws.go b/pkg/detectors/aws/aws.go
deleted file mode 100644
index a30b542f1..000000000
--- a/pkg/detectors/aws/aws.go
+++ /dev/null
@@ -1,462 +0,0 @@
-package aws
-
-import (
- "context"
- "crypto/hmac"
- "crypto/sha256"
- "encoding/hex"
- "encoding/json"
- "fmt"
- "net/http"
- "strings"
- "time"
-
- regexp "github.com/wasilibs/go-re2"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-
- "github.com/aws/aws-sdk-go/aws"
- "github.com/aws/aws-sdk-go/aws/credentials"
- "github.com/aws/aws-sdk-go/aws/session"
- "github.com/aws/aws-sdk-go/service/sns"
-)
-
-type scanner struct {
- verificationClient *http.Client
- skipIDs map[string]struct{}
- detectors.DefaultMultiPartCredentialProvider
-}
-
-// resourceTypes derived from: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
-var resourceTypes = map[string]string{
- "ABIA": "AWS STS service bearer token",
- "ACCA": "Context-specific credential",
- "AGPA": "User group",
- "AIDA": "IAM user",
- "AIPA": "Amazon EC2 instance profile",
- "AKIA": "Access key",
- "ANPA": "Managed policy",
- "ANVA": "Version in a managed policy",
- "APKA": "Public key",
- "AROA": "Role",
- "ASCA": "Certificate",
- "ASIA": "Temporary (AWS STS) access key IDs",
-}
-
-var thinkstCanaryList = map[string]struct{}{
- "052310077262": {},
- "171436882533": {},
- "534261010715": {},
- "595918472158": {},
- "717712589309": {},
- "819147034852": {},
- "992382622183": {},
- "730335385048": {},
-}
-
-const thinkstMessage = "This is an AWS canary token generated at canarytokens.org, and was not set off; learn more here: https://trufflesecurity.com/canaries"
-
-var thinkstKnockoffsCanaryList = map[string]struct{}{
- "044858866125": {},
- "251535659677": {},
- "344043088457": {},
- "351906852752": {},
- "390477818340": {},
- "426127672474": {},
- "427150556519": {},
- "439872796651": {},
- "445142720921": {},
- "465867158099": {},
- "637958123769": {},
- "693412236332": {},
- "732624840810": {},
- "735421457923": {},
- "959235150393": {},
- "982842642351": {},
-}
-
-const thinkstKnockoffsMessage = "This is an off brand AWS Canary inspired by canarytokens.org. It wasn't set off; learn more here: https://trufflesecurity.com/canaries"
-
-func New(opts ...func(*scanner)) *scanner {
- scanner := &scanner{
- skipIDs: map[string]struct{}{},
- }
- for _, opt := range opts {
-
- opt(scanner)
- }
-
- return scanner
-}
-
-func WithSkipIDs(skipIDs []string) func(*scanner) {
- return func(s *scanner) {
- ids := map[string]struct{}{}
- for _, id := range skipIDs {
- ids[id] = struct{}{}
- }
-
- s.skipIDs = ids
- }
-}
-
-// Ensure the scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*scanner)(nil)
-var _ detectors.MultiPartCredentialProvider = (*scanner)(nil)
-var _ detectors.CustomResultsCleaner = (*scanner)(nil)
-
-var (
- defaultVerificationClient = common.SaneHttpClient()
-
- // Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
- // Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
- idPat = regexp.MustCompile(`\b((AKIA|ABIA|ACCA)[0-9A-Z]{16})\b`)
- secretPat = regexp.MustCompile(`[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}`)
- // Hashes, like those for git, do technically match the secret pattern.
- // But they are extremely unlikely to be generated as an actual AWS secret.
- // So when we find them, if they're not verified, we should ignore the result.
- falsePositiveSecretCheck = regexp.MustCompile(`[a-f0-9]{40}`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s scanner) Keywords() []string {
- return []string{
- "AKIA",
- "ABIA",
- "ACCA",
- }
-}
-
-func GetHash(input string) string {
- data := []byte(input)
- hasher := sha256.New()
- hasher.Write(data)
- return hex.EncodeToString(hasher.Sum(nil))
-}
-
-func GetHMAC(key []byte, data []byte) []byte {
- hasher := hmac.New(sha256.New, key)
- hasher.Write(data)
- return hasher.Sum(nil)
-}
-
-// FromData will find and optionally verify AWS secrets in a given set of bytes.
-func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- dataStr := string(data)
-
- idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
- secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
-
- for _, idMatch := range idMatches {
- if len(idMatch) != 3 {
- continue
- }
- resIDMatch := strings.TrimSpace(idMatch[1])
-
- if s.skipIDs != nil {
- if _, ok := s.skipIDs[resIDMatch]; ok {
- continue
- }
- }
-
- for _, secretMatch := range secretMatches {
- if len(secretMatch) != 2 {
- continue
- }
- resSecretMatch := strings.TrimSpace(secretMatch[1])
-
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_AWS,
- Raw: []byte(resIDMatch),
- Redacted: resIDMatch,
- RawV2: []byte(resIDMatch + resSecretMatch),
- ExtraData: map[string]string{
- "resource_type": resourceTypes[idMatch[2]],
- },
- }
-
- account, err := common.GetAccountNumFromAWSID(resIDMatch)
- if err == nil {
- s1.ExtraData["account"] = account
- }
- if _, ok := thinkstCanaryList[account]; ok {
- s1.ExtraData["is_canary"] = "true"
- s1.ExtraData["message"] = thinkstMessage
- if verify {
- verified, arn, err := s.verifyCanary(resIDMatch, resSecretMatch)
- if verified {
- s1.Verified = true
- }
- if arn != "" {
- s1.ExtraData["arn"] = arn
- }
- if err != nil {
- s1.SetVerificationError(err, resSecretMatch)
- }
- }
- }
- if _, ok := thinkstKnockoffsCanaryList[account]; ok {
- s1.ExtraData["is_canary"] = "true"
- s1.ExtraData["message"] = thinkstKnockoffsMessage
- if verify {
- verified, arn, err := s.verifyCanary(resIDMatch, resSecretMatch)
- if verified {
- s1.Verified = true
- }
- if arn != "" {
- s1.ExtraData["arn"] = arn
- }
- if err != nil {
- s1.SetVerificationError(err, resSecretMatch)
- }
- }
- }
-
- if verify && (s1.ExtraData["is_canary"] != "true") {
- isVerified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, true)
- s1.Verified = isVerified
- // It'd be good to log when calculated account value does not match
- // the account value from verification. Should only be edge cases at most.
- // if extraData["account"] != s1.ExtraData["account"] && extraData["account"] != "" {//log here}
-
- // Append the extraData to the existing ExtraData map.
- // This will overwrite with the new verified values.
- for k, v := range extraData {
- s1.ExtraData[k] = v
- }
- if verificationErr != nil {
- s1.SetVerificationError(verificationErr, resSecretMatch)
- }
- }
-
- if !s1.Verified {
- // Unverified results that look like hashes are probably not secrets
- if falsePositiveSecretCheck.MatchString(resSecretMatch) {
- continue
- }
- }
-
- results = append(results, s1)
- // If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
- if s1.Verified {
- break
- }
- }
- }
- return results, nil
-}
-
-func (s scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool {
- return true
-}
-
-func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, retryOn403 bool) (bool, map[string]string, error) {
- // REQUEST VALUES.
- method := "GET"
- service := "sts"
- host := "sts.amazonaws.com"
- region := "us-east-1"
- endpoint := "https://sts.amazonaws.com"
- now := time.Now().UTC()
- datestamp := now.Format("20060102")
- amzDate := now.Format("20060102T150405Z0700")
-
- req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
- if err != nil {
- return false, nil, err
- }
- req.Header.Set("Accept", "application/json")
-
- // TASK 1: CREATE A CANONICAL REQUEST.
- // http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
- canonicalURI := "/"
- canonicalHeaders := "host:" + host + "\n"
- signedHeaders := "host"
- algorithm := "AWS4-HMAC-SHA256"
- credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
-
- params := req.URL.Query()
- params.Add("Action", "GetCallerIdentity")
- params.Add("Version", "2011-06-15")
- params.Add("X-Amz-Algorithm", algorithm)
- params.Add("X-Amz-Credential", resIDMatch+"/"+credentialScope)
- params.Add("X-Amz-Date", amzDate)
- params.Add("X-Amz-Expires", "30")
- params.Add("X-Amz-SignedHeaders", signedHeaders)
-
- canonicalQuerystring := params.Encode()
- payloadHash := GetHash("") // empty payload
- canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
-
- // TASK 2: CREATE THE STRING TO SIGN.
- stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
-
- // TASK 3: CALCULATE THE SIGNATURE.
- // https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
- hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
- hash = GetHMAC(hash, []byte(region))
- hash = GetHMAC(hash, []byte(service))
- hash = GetHMAC(hash, []byte("aws4_request"))
-
- signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
- signature := hex.EncodeToString(signature2)
-
- // TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
- params.Add("X-Amz-Signature", signature)
- req.Header.Add("Content-type", "application/x-www-form-urlencoded; charset=utf-8")
- req.URL.RawQuery = params.Encode()
-
- client := s.verificationClient
- if client == nil {
- client = defaultVerificationClient
- }
-
- extraData := map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
- }
-
- res, err := client.Do(req)
- if err == nil {
- defer res.Body.Close()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- identityInfo := identityRes{}
- err := json.NewDecoder(res.Body).Decode(&identityInfo)
- if err == nil {
- extraData["account"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account
- extraData["user_id"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID
- extraData["arn"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn
- return true, extraData, nil
- } else {
- return false, nil, err
- }
- } else if res.StatusCode == 403 {
- // Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
- // share a key ID but are signed with different secrets the second one will be rejected with a 403 that
- // carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
- // valid. Since this is exactly our access pattern, we need to work around it.
- //
- // Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
- // response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
- // the response to the original second request, or read to its end, or the resubmission will also yield a
- // SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
- // SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
- // after decoding its response body, which requires reading the entire response body, which disables the
- // workaround.
- //
- // We are clearly deep in the guts of AWS implementation details here, so this all might change with no
- // notice. If you're here because something in this detector broke, you have my condolences.
- if retryOn403 {
- return s.verifyMatch(ctx, resIDMatch, resSecretMatch, false)
- }
- var body awsErrorResponseBody
- err = json.NewDecoder(res.Body).Decode(&body)
- if err == nil {
- // All instances of the code I've seen in the wild are PascalCased but this check is
- // case-insensitive out of an abundance of caution
- if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
- return false, nil, nil
- } else {
- return false, nil, fmt.Errorf("request returned status %d with an unexpected reason (%s: %s)", res.StatusCode, body.Error.Code, body.Error.Message)
- }
- } else {
- return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
- }
- } else {
- return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
- }
- } else {
- return false, nil, err
- }
-}
-
-func (s scanner) verifyCanary(resIDMatch, resSecretMatch string) (bool, string, error) {
- // Prep AWS Creds for SNS
- sess := session.Must(session.NewSession(&aws.Config{
- Region: aws.String("us-east-1"), // any region seems to work
- Credentials: credentials.NewStaticCredentials(
- resIDMatch,
- resSecretMatch,
- "",
- ),
- HTTPClient: s.verificationClient,
- }))
- svc := sns.New(sess)
-
- // Prep vars and Publish to SNS
- _, err := svc.Publish(&sns.PublishInput{
- Message: aws.String("foo"),
- PhoneNumber: aws.String("1"),
- })
-
- if strings.Contains(err.Error(), "not authorized to perform") {
- arn := strings.Split(err.Error(), "User: ")[1]
- arn = strings.Split(arn, " is not authorized to perform: ")[0]
- return true, arn, nil
- } else if strings.Contains(err.Error(), "does not match the signature you provided") {
- return false, "", nil
- } else if strings.Contains(err.Error(), "status code: 403") {
- return false, "", nil
- } else {
- return false, "", err
- }
-}
-
-func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {
- if len(results) == 0 {
- return results
- }
-
- // For every ID, we want at most one result, preferably verified.
- idResults := map[string]detectors.Result{}
- for _, result := range results {
- // Always accept the verified result as the result for the given ID.
- if result.Verified {
- idResults[result.Redacted] = result
- continue
- }
-
- // Only include an unverified result if we don't already have a result for a given ID.
- if _, exist := idResults[result.Redacted]; !exist {
- idResults[result.Redacted] = result
- }
- }
-
- var out []detectors.Result
- for _, r := range idResults {
- out = append(out, r)
- }
- return out
-}
-
-type awsError struct {
- Code string `json:"Code"`
- Message string `json:"Message"`
-}
-
-type awsErrorResponseBody struct {
- Error awsError `json:"Error"`
-}
-
-type identityRes struct {
- GetCallerIdentityResponse struct {
- GetCallerIdentityResult struct {
- Account string `json:"Account"`
- Arn string `json:"Arn"`
- UserID string `json:"UserId"`
- } `json:"GetCallerIdentityResult"`
- ResponseMetadata struct {
- RequestID string `json:"RequestId"`
- } `json:"ResponseMetadata"`
- } `json:"GetCallerIdentityResponse"`
-}
-
-func (s scanner) Type() detectorspb.DetectorType {
- return detectorspb.DetectorType_AWS
-}
-
-func (s scanner) Description() string {
- return "AWS (Amazon Web Services) is a comprehensive cloud computing platform offering a wide range of on-demand services like computing power, storage, databases. API keys for AWS can have varying amount of access to these services depending on the IAM policy attached."
-}
diff --git a/pkg/detectors/aws/common.go b/pkg/detectors/aws/common.go
new file mode 100644
index 000000000..c18addc0e
--- /dev/null
+++ b/pkg/detectors/aws/common.go
@@ -0,0 +1,32 @@
+package aws
+
+import regexp "github.com/wasilibs/go-re2"
+
+const (
+ RequiredIdEntropy = 3.0
+ RequiredSecretEntropy = 4.25
+)
+
+var SecretPat = regexp.MustCompile(`(?:[^A-Za-z0-9+/]|\A)([A-Za-z0-9+/]{40})(?:[^A-Za-z0-9+/]|\z)`)
+
+type IdentityResponse struct {
+ GetCallerIdentityResponse struct {
+ GetCallerIdentityResult struct {
+ Account string `json:"Account"`
+ Arn string `json:"Arn"`
+ UserID string `json:"UserId"`
+ } `json:"GetCallerIdentityResult"`
+ ResponseMetadata struct {
+ RequestID string `json:"RequestId"`
+ } `json:"ResponseMetadata"`
+ } `json:"GetCallerIdentityResponse"`
+}
+
+type Error struct {
+ Code string `json:"Code"`
+ Message string `json:"Message"`
+}
+
+type ErrorResponseBody struct {
+ Error Error `json:"Error"`
+}
diff --git a/pkg/detectors/aws/session_keys/sessionkey.go b/pkg/detectors/aws/session_keys/sessionkey.go
new file mode 100644
index 000000000..debb002a2
--- /dev/null
+++ b/pkg/detectors/aws/session_keys/sessionkey.go
@@ -0,0 +1,306 @@
+package session_keys
+
+import (
+ "context"
+ "encoding/hex"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "strings"
+ "time"
+
+ regexp "github.com/wasilibs/go-re2"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ logContext "github.com/trufflesecurity/trufflehog/v3/pkg/context"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+type scanner struct {
+ *detectors.CustomMultiPartCredentialProvider
+ verificationClient *http.Client
+ skipIDs map[string]struct{}
+}
+
+func New(opts ...func(*scanner)) *scanner {
+ scanner := &scanner{
+ skipIDs: map[string]struct{}{},
+ }
+ for _, opt := range opts {
+ opt(scanner)
+ }
+
+ scanner.CustomMultiPartCredentialProvider = detectors.NewCustomMultiPartCredentialProvider(2048) // ????
+ return scanner
+}
+
+func WithSkipIDs(skipIDs []string) func(*scanner) {
+ return func(s *scanner) {
+ ids := map[string]struct{}{}
+ for _, id := range skipIDs {
+ ids[id] = struct{}{}
+ }
+
+ s.skipIDs = ids
+ }
+}
+
+// Ensure the scanner satisfies the interface at compile time.
+var _ interface {
+ detectors.Detector
+ detectors.CustomResultsCleaner
+} = (*scanner)(nil)
+
+var (
+ defaultVerificationClient = common.SaneHttpClient()
+
+ // Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
+ idPat = regexp.MustCompile(`\b((?:ASIA)[A-Z0-9]{16})\b`)
+ sessionPat = regexp.MustCompile(`(?:[^A-Za-z0-9+/]|\A)([a-zA-Z0-9+/]{100,}={0,3})(?:[^A-Za-z0-9+/=]|\z)`)
+)
+
+// Keywords are used for efficiently pre-filtering chunks.
+// Use identifiers in the secret preferably, or the provider name.
+func (s scanner) Keywords() []string {
+ return []string{"ASIA"}
+}
+
+// FromData will find and optionally verify AWS secrets in a given set of bytes.
+func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ logger := logContext.AddLogger(ctx).Logger().WithName("awssessionkey")
+ dataStr := string(data)
+ dataStr = aws.UrlEncodedReplacer.Replace(dataStr)
+
+ // Filter & deduplicate matches.
+ idMatches := make(map[string]struct{})
+ for _, matches := range idPat.FindAllStringSubmatch(dataStr, -1) {
+ idMatches[matches[1]] = struct{}{}
+ }
+ secretMatches := make(map[string]struct{})
+ for _, matches := range aws.SecretPat.FindAllStringSubmatch(dataStr, -1) {
+ secretMatches[matches[1]] = struct{}{}
+ }
+ sessionMatches := make(map[string]struct{})
+ for _, matches := range sessionPat.FindAllStringSubmatch(dataStr, -1) {
+ sessionMatches[matches[1]] = struct{}{}
+ }
+
+ // Process matches.
+ for idMatch := range idMatches {
+ if detectors.StringShannonEntropy(idMatch) < aws.RequiredIdEntropy {
+ continue
+ }
+ if s.skipIDs != nil {
+ if _, ok := s.skipIDs[idMatch]; ok {
+ continue
+ }
+ }
+
+ for secretMatch := range secretMatches {
+ if detectors.StringShannonEntropy(secretMatch) < aws.RequiredSecretEntropy {
+ continue
+ }
+
+ for sessionMatch := range sessionMatches {
+ if detectors.StringShannonEntropy(sessionMatch) < 4.5 {
+ continue
+ }
+ if !checkSessionToken(sessionMatch, secretMatch) {
+ continue
+ }
+
+ s1 := detectors.Result{
+ DetectorType: detectorspb.DetectorType_AWSSessionKey,
+ Raw: []byte(idMatch),
+ RawV2: []byte(fmt.Sprintf("%s:%s:%s", idMatch, secretMatch, sessionMatch)),
+ Redacted: idMatch,
+ ExtraData: make(map[string]string),
+ }
+
+ if verify {
+ isVerified, extraData, verificationErr := s.verifyMatch(ctx, idMatch, secretMatch, sessionMatch, true)
+ s1.Verified = isVerified
+ if extraData != nil {
+ s1.ExtraData = extraData
+ }
+ s1.SetVerificationError(verificationErr, secretMatch)
+ }
+
+ if !s1.Verified && aws.FalsePositiveSecretPat.MatchString(secretMatch) {
+ // Unverified results that look like hashes are probably not secrets
+ continue
+ }
+
+ // If we haven't already found an account number for this ID (via API), calculate one.
+ if _, ok := s1.ExtraData["account"]; !ok {
+ if account, err := aws.GetAccountNumFromID(idMatch); err != nil {
+ logger.V(3).Info("Failed to decode account number", "err", err)
+ } else {
+ s1.ExtraData["account"] = account
+ }
+ }
+
+ results = append(results, s1)
+ // If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
+ if s1.Verified {
+ delete(sessionMatches, secretMatch)
+ delete(sessionMatches, sessionMatch)
+ break
+ }
+ }
+ }
+ }
+ return results, nil
+}
+
+func (s scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool {
+ return true
+}
+
+const (
+ method = "GET"
+ service = "sts"
+ host = "sts.amazonaws.com"
+ region = "us-east-1"
+ endpoint = "https://sts.amazonaws.com"
+)
+
+func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, resSessionMatch string, retryOn403 bool) (bool, map[string]string, error) {
+ // REQUEST VALUES.
+ now := time.Now().UTC()
+ datestamp := now.Format("20060102")
+ amzDate := now.Format("20060102T150405Z")
+
+ req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
+ if err != nil {
+ return false, nil, err
+ }
+ req.Header.Set("Accept", "application/json")
+
+ // TASK 1: CREATE A CANONICAL REQUEST.
+ // http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html
+ canonicalURI := "/"
+ canonicalHeaders := "host:" + host + "\n" + "x-amz-date:" + amzDate + "\n" + "x-amz-security-token:" + resSessionMatch + "\n"
+ signedHeaders := "host;x-amz-date;x-amz-security-token"
+ algorithm := "AWS4-HMAC-SHA256"
+ credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
+
+ params := req.URL.Query()
+ params.Add("Action", "GetCallerIdentity")
+ params.Add("Version", "2011-06-15")
+ canonicalQuerystring := params.Encode()
+ payloadHash := aws.GetHash("") // empty payload
+ canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
+
+ // TASK 2: CREATE THE STRING TO SIGN.
+ stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + aws.GetHash(canonicalRequest)
+
+ // TASK 3: CALCULATE THE SIGNATURE.
+ // https://docs.aws.amazon.com/general/latest/gr/sigv4-calculate-signature.html
+ hash := aws.GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
+ hash = aws.GetHMAC(hash, []byte(region))
+ hash = aws.GetHMAC(hash, []byte(service))
+ hash = aws.GetHMAC(hash, []byte("aws4_request"))
+
+ signature2 := aws.GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
+ signature := hex.EncodeToString(signature2)
+
+ // TASK 4: ADD SIGNING INFORMATION TO THE REQUEST.
+ authorizationHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s",
+ algorithm, resIDMatch, credentialScope, signedHeaders, signature)
+
+ req.Header.Add("Authorization", authorizationHeader)
+ req.Header.Add("x-amz-date", amzDate)
+ req.Header.Add("x-amz-security-token", resSessionMatch)
+
+ req.URL.RawQuery = params.Encode()
+
+ client := s.verificationClient
+ if client == nil {
+ client = defaultVerificationClient
+ }
+
+ res, err := client.Do(req)
+ if err != nil {
+ return false, nil, err
+ }
+ defer func() {
+ _, _ = io.Copy(io.Discard, res.Body)
+ _ = res.Body.Close()
+ }()
+
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ identityInfo := aws.IdentityResponse{}
+ if err := json.NewDecoder(res.Body).Decode(&identityInfo); err != nil {
+ return false, nil, err
+ }
+
+ extraData := map[string]string{
+ "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
+ "account": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account,
+ "user_id": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID,
+ "arn": identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn,
+ }
+ return true, extraData, nil
+ } else if res.StatusCode == 403 {
+ // Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
+ // share a key ID but are signed with different secrets the second one will be rejected with a 403 that
+ // carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
+ // valid. Since this is exactly our access pattern, we need to work around it.
+ //
+ // Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
+ // response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
+ // the response to the original second request, or read to its end, or the resubmission will also yield a
+ // SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
+ // SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
+ // after decoding its response body, which requires reading the entire response body, which disables the
+ // workaround.
+ //
+ // We are clearly deep in the guts of AWS implementation details here, so this all might change with no
+ // notice. If you're here because something in this detector broke, you have my condolences.
+ if retryOn403 {
+ return s.verifyMatch(ctx, resIDMatch, resSecretMatch, resSessionMatch, false)
+ }
+
+ var body aws.ErrorResponseBody
+ if err = json.NewDecoder(res.Body).Decode(&body); err != nil {
+ return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
+ }
+ // All instances of the code I've seen in the wild are PascalCased but this check is
+ // case-insensitive out of an abundance of caution
+ if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
+ return false, nil, nil
+ } else if strings.EqualFold(body.Error.Code, "ExpiredToken") {
+ // ExpiredToken: The security token included in the request is expired
+ return false, nil, nil
+ }
+ return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
+ } else {
+ return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
+ }
+}
+
+func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {
+ return aws.CleanResults(results)
+}
+
+// Reference: https://nitter.poast.org/TalBeerySec/status/1816449053841838223#m
+func checkSessionToken(sessionToken string, secret string) bool {
+ if !(strings.Contains(sessionToken, "YXdz") || strings.Contains(sessionToken, "Jb3JpZ2luX2Vj")) ||
+ strings.Contains(sessionToken, secret) {
+ // Handle error if the sessionToken is not a valid base64 string
+ return false
+ }
+ return true
+}
+
+func (s scanner) Type() detectorspb.DetectorType {
+ return detectorspb.DetectorType_AWSSessionKey
+}
+
+func (s scanner) Description() string {
+ return "AWS (Amazon Web Services) is a comprehensive cloud computing platform offering a wide range of on-demand services like computing power, storage, databases. API keys for AWS can have varying amount of access to these services depending on the IAM policy attached. AWS Session Tokens are short-lived keys."
+}
diff --git a/pkg/detectors/awssessionkeys/awssessionkeys_test.go b/pkg/detectors/aws/session_keys/sessionkeys_test.go
similarity index 99%
rename from pkg/detectors/awssessionkeys/awssessionkeys_test.go
rename to pkg/detectors/aws/session_keys/sessionkeys_test.go
index 8ed4028e6..ee6cf4f7f 100644
--- a/pkg/detectors/awssessionkeys/awssessionkeys_test.go
+++ b/pkg/detectors/aws/session_keys/sessionkeys_test.go
@@ -1,4 +1,4 @@
-package awssessionkey
+package session_keys
import (
"context"
diff --git a/pkg/detectors/aws/utils.go b/pkg/detectors/aws/utils.go
new file mode 100644
index 000000000..468562b55
--- /dev/null
+++ b/pkg/detectors/aws/utils.go
@@ -0,0 +1,114 @@
+package aws
+
+import (
+ "crypto/hmac"
+ "crypto/sha256"
+ "encoding/base32"
+ "encoding/binary"
+ "encoding/hex"
+ "fmt"
+ "strings"
+
+ regexp "github.com/wasilibs/go-re2"
+
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+)
+
+// ResourceTypes derived from: https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
+var ResourceTypes = map[string]string{
+ "ABIA": "AWS STS service bearer token",
+ "ACCA": "Context-specific credential",
+ "AGPA": "User group",
+ "AIDA": "IAM user",
+ "AIPA": "Amazon EC2 instance profile",
+ "AKIA": "Access key",
+ "ANPA": "Managed policy",
+ "ANVA": "Version in a managed policy",
+ "APKA": "Public key",
+ "AROA": "Role",
+ "ASCA": "Certificate",
+ "ASIA": "Temporary (AWS STS) access key IDs",
+}
+
+// UrlEncodedReplacer helps capture base64-encoded results that may be url-encoded.
+// TODO: Add this as a decoder, or make it a more generic.
+var UrlEncodedReplacer = strings.NewReplacer(
+ "%2B", "+",
+ "%2b", "+",
+ "%2F", "/",
+ "%2f", "/",
+ "%3d", "=",
+ "%3D", "=",
+)
+
+// Hashes, like those for git, do technically match the secret pattern.
+// But they are extremely unlikely to be generated as an actual AWS secret.
+// So when we find them, if they're not verified, we should ignore the result.
+var FalsePositiveSecretPat = regexp.MustCompile(`[a-f0-9]{40}`)
+
+func GetAccountNumFromID(id string) (string, error) {
+ // Function to get the account number from an AWS ID (no verification required)
+ // Source: https://medium.com/@TalBeerySec/a-short-note-on-aws-key-id-f88cc4317489
+ if len(id) < 4 {
+ return "", fmt.Errorf("AWSID is too short")
+ }
+ if id[4] == 'I' || id[4] == 'J' {
+ return "", fmt.Errorf("can't get account number from AKIAJ/ASIAJ or AKIAI/ASIAI keys")
+ }
+ trimmedAWSID := id[4:]
+ decodedBytes, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(trimmedAWSID))
+ if err != nil {
+ return "", err
+ }
+
+ if len(decodedBytes) < 6 {
+ return "", fmt.Errorf("decoded AWSID is too short")
+ }
+
+ data := make([]byte, 8)
+ copy(data[2:], decodedBytes[0:6])
+ z := binary.BigEndian.Uint64(data)
+ const mask uint64 = 0x7fffffffff80
+ accountNum := (z & mask) >> 7
+ return fmt.Sprintf("%012d", accountNum), nil
+}
+
+func GetHash(input string) string {
+ data := []byte(input)
+ hasher := sha256.New()
+ hasher.Write(data)
+ return hex.EncodeToString(hasher.Sum(nil))
+}
+
+func GetHMAC(key []byte, data []byte) []byte {
+ hasher := hmac.New(sha256.New, key)
+ hasher.Write(data)
+ return hasher.Sum(nil)
+}
+
+func CleanResults(results []detectors.Result) []detectors.Result {
+ if len(results) == 0 {
+ return results
+ }
+
+ // For every ID, we want at most one result, preferably verified.
+ idResults := map[string]detectors.Result{}
+ for _, result := range results {
+ // Always accept the verified result as the result for the given ID.
+ if result.Verified {
+ idResults[result.Redacted] = result
+ continue
+ }
+
+ // Only include an unverified result if we don't already have a result for a given ID.
+ if _, exist := idResults[result.Redacted]; !exist {
+ idResults[result.Redacted] = result
+ }
+ }
+
+ var out []detectors.Result
+ for _, r := range idResults {
+ out = append(out, r)
+ }
+ return out
+}
diff --git a/pkg/detectors/awssessionkeys/awssessionkey.go b/pkg/detectors/awssessionkeys/awssessionkey.go
deleted file mode 100644
index 51f5d51c9..000000000
--- a/pkg/detectors/awssessionkeys/awssessionkey.go
+++ /dev/null
@@ -1,348 +0,0 @@
-package awssessionkey
-
-import (
- "context"
- "crypto/hmac"
- "crypto/sha256"
- "encoding/hex"
- "encoding/json"
- "fmt"
- "net/http"
- "strings"
- "time"
-
- regexp "github.com/wasilibs/go-re2"
-
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
- "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
-)
-
-type scanner struct {
- detectors.DefaultMultiPartCredentialProvider
- verificationClient *http.Client
- skipIDs map[string]struct{}
-}
-
-func New(opts ...func(*scanner)) *scanner {
- scanner := &scanner{
- skipIDs: map[string]struct{}{},
- }
- for _, opt := range opts {
-
- opt(scanner)
- }
-
- return scanner
-}
-
-func WithSkipIDs(skipIDs []string) func(*scanner) {
- return func(s *scanner) {
- ids := map[string]struct{}{}
- for _, id := range skipIDs {
- ids[id] = struct{}{}
- }
-
- s.skipIDs = ids
- }
-}
-
-// Ensure the scanner satisfies the interface at compile time.
-var _ detectors.Detector = (*scanner)(nil)
-var _ detectors.CustomResultsCleaner = (*scanner)(nil)
-
-var (
- defaultVerificationClient = common.SaneHttpClient()
-
- // Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
- // Key types are from this list https://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html#identifiers-unique-ids
- idPat = regexp.MustCompile(`\b((?:ASIA)[0-9A-Z]{16})\b`)
- secretPat = regexp.MustCompile(`\b[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+\/]{40})[^A-Za-z0-9+\/]{0,1}\b`)
- sessionPat = regexp.MustCompile(`\b[^A-Za-z0-9+\/]{0,1}([A-Za-z0-9+=\/]{41,1000})[^A-Za-z0-9+=\/]{0,1}\b`)
- // Hashes, like those for git, do technically match the secret pattern.
- // But they are extremely unlikely to be generated as an actual AWS secret.
- // So when we find them, if they're not verified, we should ignore the result.
- falsePositiveSecretCheck = regexp.MustCompile(`[a-f0-9]{40}`)
-)
-
-// Keywords are used for efficiently pre-filtering chunks.
-// Use identifiers in the secret preferably, or the provider name.
-func (s scanner) Keywords() []string {
- return []string{
- "ASIA",
- }
-}
-
-func GetHash(input string) string {
- data := []byte(input)
- hasher := sha256.New()
- hasher.Write(data)
- return hex.EncodeToString(hasher.Sum(nil))
-}
-
-func GetHMAC(key []byte, data []byte) []byte {
- hasher := hmac.New(sha256.New, key)
- hasher.Write(data)
- return hasher.Sum(nil)
-}
-
-// Reference: https://nitter.poast.org/TalBeerySec/status/1816449053841838223#m
-func checkSessionToken(sessionToken string, secret string) bool {
- if !(strings.Contains(sessionToken, "YXdz") || strings.Contains(sessionToken, "Jb3JpZ2luX2Vj")) ||
- strings.Contains(sessionToken, secret) {
- // Handle error if the sessionToken is not a valid base64 string
- return false
- }
- return true
-}
-
-// FromData will find and optionally verify AWS secrets in a given set of bytes.
-func (s scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- dataStr := string(data)
-
- idMatches := idPat.FindAllStringSubmatch(dataStr, -1)
- secretMatches := secretPat.FindAllStringSubmatch(dataStr, -1)
- sessionMatches := sessionPat.FindAllStringSubmatch(dataStr, -1)
-
- for _, idMatch := range idMatches {
- if len(idMatch) != 2 {
- continue
- }
- resIDMatch := strings.TrimSpace(idMatch[1])
-
- if s.skipIDs != nil {
- if _, ok := s.skipIDs[resIDMatch]; ok {
- continue
- }
- }
-
- for _, secretMatch := range secretMatches {
- if len(secretMatch) != 2 {
- continue
- }
- resSecretMatch := strings.TrimSpace(secretMatch[1])
-
- for _, sessionMatch := range sessionMatches {
- if len(sessionMatch) != 2 {
- continue
- }
- resSessionMatch := strings.TrimSpace(sessionMatch[1])
- if !checkSessionToken(resSessionMatch, resSecretMatch) {
- continue
- }
- s1 := detectors.Result{
- DetectorType: detectorspb.DetectorType_AWSSessionKey,
- Raw: []byte(resIDMatch),
- Redacted: resIDMatch,
- RawV2: []byte(resIDMatch + resSecretMatch + resSessionMatch),
- ExtraData: make(map[string]string),
- }
-
- if verify {
- isVerified, extraData, verificationErr := s.verifyMatch(ctx, resIDMatch, resSecretMatch, resSessionMatch, true)
- s1.Verified = isVerified
- if extraData != nil {
- s1.ExtraData = extraData
- }
- s1.SetVerificationError(verificationErr, resSecretMatch)
- }
-
- if !s1.Verified {
- // Unverified results that look like hashes are probably not secrets
- if falsePositiveSecretCheck.MatchString(resSecretMatch) {
- continue
- }
- }
-
- // If we haven't already found an account number for this ID (via API), calculate one.
- if _, ok := s1.ExtraData["account"]; !ok {
- account, err := common.GetAccountNumFromAWSID(resIDMatch)
- if err == nil {
- s1.ExtraData["account"] = account
- }
- }
-
- results = append(results, s1)
- // If we've found a verified match with this ID, we don't need to look for any more. So move on to the next ID.
- if s1.Verified {
- break
- }
- }
- }
- }
- return results, nil
-}
-
-func (s scanner) ShouldCleanResultsIrrespectiveOfConfiguration() bool {
- return true
-}
-
-func (s scanner) verifyMatch(ctx context.Context, resIDMatch, resSecretMatch string, resSessionMatch string, retryOn403 bool) (bool, map[string]string, error) {
-
- // REQUEST VALUES.
- method := "GET"
- service := "sts"
- host := "sts.amazonaws.com"
- region := "us-east-1"
- endpoint := "https://sts.amazonaws.com"
- now := time.Now().UTC()
- datestamp := now.Format("20060102")
- amzDate := now.Format("20060102T150405Z")
-
- req, err := http.NewRequestWithContext(ctx, method, endpoint, nil)
- if err != nil {
- return false, nil, err
- }
- req.Header.Set("Accept", "application/json")
-
- canonicalURI := "/"
- canonicalHeaders := "host:" + host + "\n" + "x-amz-date:" + amzDate + "\n" + "x-amz-security-token:" + resSessionMatch + "\n"
- signedHeaders := "host;x-amz-date;x-amz-security-token"
- algorithm := "AWS4-HMAC-SHA256"
- credentialScope := fmt.Sprintf("%s/%s/%s/aws4_request", datestamp, region, service)
-
- params := req.URL.Query()
- params.Add("Action", "GetCallerIdentity")
- params.Add("Version", "2011-06-15")
- canonicalQuerystring := params.Encode()
- payloadHash := GetHash("") // empty payload
- canonicalRequest := method + "\n" + canonicalURI + "\n" + canonicalQuerystring + "\n" + canonicalHeaders + "\n" + signedHeaders + "\n" + payloadHash
-
- stringToSign := algorithm + "\n" + amzDate + "\n" + credentialScope + "\n" + GetHash(canonicalRequest)
-
- hash := GetHMAC([]byte(fmt.Sprintf("AWS4%s", resSecretMatch)), []byte(datestamp))
- hash = GetHMAC(hash, []byte(region))
- hash = GetHMAC(hash, []byte(service))
- hash = GetHMAC(hash, []byte("aws4_request"))
-
- signature2 := GetHMAC(hash, []byte(stringToSign)) // Get Signature HMAC SHA256
- signature := hex.EncodeToString(signature2)
-
- authorizationHeader := fmt.Sprintf("%s Credential=%s/%s, SignedHeaders=%s, Signature=%s",
- algorithm, resIDMatch, credentialScope, signedHeaders, signature)
-
- req.Header.Add("Authorization", authorizationHeader)
- req.Header.Add("x-amz-date", amzDate)
- req.Header.Add("x-amz-security-token", resSessionMatch)
-
- req.URL.RawQuery = params.Encode()
-
- client := s.verificationClient
- if client == nil {
- client = defaultVerificationClient
- }
-
- extraData := map[string]string{
- "rotation_guide": "https://howtorotate.com/docs/tutorials/aws/",
- }
-
- res, err := client.Do(req)
- if err == nil {
- defer res.Body.Close()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- identityInfo := identityRes{}
- err := json.NewDecoder(res.Body).Decode(&identityInfo)
- if err == nil {
- extraData["account"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Account
- extraData["user_id"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.UserID
- extraData["arn"] = identityInfo.GetCallerIdentityResponse.GetCallerIdentityResult.Arn
- return true, extraData, nil
- } else {
- return false, nil, err
- }
- } else if res.StatusCode == 403 {
- // Experimentation has indicated that if you make two GetCallerIdentity requests within five seconds that
- // share a key ID but are signed with different secrets the second one will be rejected with a 403 that
- // carries a SignatureDoesNotMatch code in its body. This happens even if the second ID-secret pair is
- // valid. Since this is exactly our access pattern, we need to work around it.
- //
- // Fortunately, experimentation has also revealed a workaround: simply resubmit the second request. The
- // response to the resubmission will be as expected. But there's a caveat: You can't have closed the body of
- // the response to the original second request, or read to its end, or the resubmission will also yield a
- // SignatureDoesNotMatch. For this reason, we have to re-request all 403s. We can't re-request only
- // SignatureDoesNotMatch responses, because we can only tell whether a given 403 is a SignatureDoesNotMatch
- // after decoding its response body, which requires reading the entire response body, which disables the
- // workaround.
- //
- // We are clearly deep in the guts of AWS implementation details here, so this all might change with no
- // notice. If you're here because something in this detector broke, you have my condolences.
- if retryOn403 {
- return s.verifyMatch(ctx, resIDMatch, resSecretMatch, resSessionMatch, false)
- }
- var body awsErrorResponseBody
- err = json.NewDecoder(res.Body).Decode(&body)
- if err == nil {
- // All instances of the code I've seen in the wild are PascalCased but this check is
- // case-insensitive out of an abundance of caution
- if strings.EqualFold(body.Error.Code, "InvalidClientTokenId") {
- return false, nil, nil
- } else {
- return false, nil, fmt.Errorf("request to %v returned status %d with an unexpected reason (%s: %s)", res.Request.URL, res.StatusCode, body.Error.Code, body.Error.Message)
- }
- } else {
- return false, nil, fmt.Errorf("couldn't parse the sts response body (%v)", err)
- }
- } else {
- return false, nil, fmt.Errorf("request to %v returned unexpected status %d", res.Request.URL, res.StatusCode)
- }
- } else {
- return false, nil, err
- }
-}
-
-func (s scanner) CleanResults(results []detectors.Result) []detectors.Result {
- if len(results) == 0 {
- return results
- }
-
- // For every ID, we want at most one result, preferably verified.
- idResults := map[string]detectors.Result{}
- for _, result := range results {
- // Always accept the verified result as the result for the given ID.
- if result.Verified {
- idResults[result.Redacted] = result
- continue
- }
-
- // Only include an unverified result if we don't already have a result for a given ID.
- if _, exist := idResults[result.Redacted]; !exist {
- idResults[result.Redacted] = result
- }
- }
-
- var out []detectors.Result
- for _, r := range idResults {
- out = append(out, r)
- }
- return out
-}
-
-type awsError struct {
- Code string `json:"Code"`
- Message string `json:"Message"`
-}
-
-type awsErrorResponseBody struct {
- Error awsError `json:"Error"`
-}
-
-type identityRes struct {
- GetCallerIdentityResponse struct {
- GetCallerIdentityResult struct {
- Account string `json:"Account"`
- Arn string `json:"Arn"`
- UserID string `json:"UserId"`
- } `json:"GetCallerIdentityResult"`
- ResponseMetadata struct {
- RequestID string `json:"RequestId"`
- } `json:"ResponseMetadata"`
- } `json:"GetCallerIdentityResponse"`
-}
-
-func (s scanner) Type() detectorspb.DetectorType {
- return detectorspb.DetectorType_AWSSessionKey
-}
-
-func (s scanner) Description() string {
- return "AWS (Amazon Web Services) is a comprehensive cloud computing platform offering a wide range of on-demand services like computing power, storage, databases. API keys for AWS can have varying amount of access to these services depending on the IAM policy attached. AWS Session Tokens are short-lived keys."
-}
diff --git a/pkg/engine/defaults/defaults.go b/pkg/engine/defaults/defaults.go
index 71542e183..5cae1a292 100644
--- a/pkg/engine/defaults/defaults.go
+++ b/pkg/engine/defaults/defaults.go
@@ -63,8 +63,8 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/autopilot"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/avazapersonalaccesstoken"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aviationstack"
- "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws"
- awssessionkey "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/awssessionkeys"
+ aws_access_keys "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws/access_keys"
+ aws_session_keys "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aws/session_keys"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/axonaut"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/aylien"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ayrshare"
@@ -882,8 +882,8 @@ func buildDetectorList() []detectors.Detector {
&autopilot.Scanner{},
&avazapersonalaccesstoken.Scanner{},
&aviationstack.Scanner{},
- aws.New(),
- awssessionkey.New(),
+ aws_access_keys.New(),
+ aws_session_keys.New(),
&axonaut.Scanner{},
&aylien.Scanner{},
&ayrshare.Scanner{},