Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
fixed and improved squareapp detector (#3993)
Kashif Khan0f360b073618
diff --git a/pkg/detectors/squareapp/squareapp.go b/pkg/detectors/squareapp/squareapp.goindex 021bbf228..5f214d1ef 100644--- a/pkg/detectors/squareapp/squareapp.go+++ b/pkg/detectors/squareapp/squareapp.go@@ -5,7 +5,9 @@ import ("context""encoding/json""fmt"+ "io""net/http"+ "strings"regexp "github.com/wasilibs/go-re2"@@ -14,7 +16,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{+type Scanner struct {detectors.DefaultMultiPartCredentialProvider}@@ -22,10 +24,17 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- // possibly always `sq0csp` for secret- // and `sq0idb` for app- keyPat = regexp.MustCompile(`[\w\-]*sq0i[a-z]{2}-[0-9A-Za-z\-_]{22,43}`)- secPat = regexp.MustCompile(`[\w\-]*sq0c[a-z]{2}-[0-9A-Za-z\-_]{40,50}`)+ client = common.SaneHttpClient()+ /*+ The sandbox id and secret has word `sandbox-` as prefix+ possibly always `sq0csp` for secret and `sq0idb` for app+ */+ keyPat = regexp.MustCompile(`(?:sandbox-)?sq0i[a-z]{2}-[0-9A-Za-z_-]{22,43}`)+ secPat = regexp.MustCompile(`(?:sandbox-)?sq0c[a-z]{2}-[0-9A-Za-z_-]{40,50}`)++ // api endpoints+ sandboxEndpoint = "https://connect.squareupsandbox.com/oauth2/revoke"+ prodEndpoint = "https://connect.squareup.com/oauth2/revoke")// Keywords are used for efficiently pre-filtering chunks.@@ -34,61 +43,114 @@ func (s Scanner) Keywords() []string {return []string{"sq0i"}}+func (s Scanner) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType_SquareApp+}++func (s Scanner) Description() string {+ return "Square is a financial services and mobile payment company. Square credentials can be used to access and manage payment processing and other financial services."+}+// FromData will find and optionally verify SquareApp secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)- matches := keyPat.FindAllString(dataStr, -1)- secMatches := secPat.FindAllString(dataStr, -1)- for _, match := range matches {- for _, secMatch := range secMatches {+ var uniqueIDMatches, uniqueSecretMatches = make(map[string]struct{}), make(map[string]struct{})++ for _, match := range keyPat.FindAllString(dataStr, -1) {+ uniqueIDMatches[match] = struct{}{}+ }++ for _, match := range secPat.FindAllString(dataStr, -1) {+ uniqueSecretMatches[match] = struct{}{}+ }++ for id := range uniqueIDMatches {+ for secret := range uniqueSecretMatches {+ // if both are not from same env, continue+ if !hasSamePrefix(id, secret) {+ continue+ }result := detectors.Result{DetectorType: detectorspb.DetectorType_SquareApp,- Raw: []byte(match),- Redacted: match,+ Raw: []byte(id),+ Redacted: id,+ ExtraData: map[string]string{},}- if verify {- baseURL := "https://connect.squareupsandbox.com/oauth2/revoke"-- client := common.SaneHttpClient()- reqData, err := json.Marshal(map[string]string{- "client_id": match,- "access_token": "fakeTruffleHogAccessTokenForVerification",- })- if err != nil {- return results, err- }-- req, err := http.NewRequestWithContext(ctx, "POST", baseURL, bytes.NewReader(reqData))- if err != nil {- continue- }- req.Header.Add("Authorization", fmt.Sprintf("Client %s", secMatch))- req.Header.Add("Content-Type", "application/json")-- res, err := client.Do(req)- if err == nil {- res.Body.Close() // The request body is unused.-- // 404 = Correct credentials. The fake access token should not be found.- if res.StatusCode == http.StatusNotFound {- result.Verified = true- }- }+ var isVerified bool+ var verificationErr error++ // verify against sandbox endpoint+ if verify && isSandbox(id) {+ isVerified, verificationErr = verifySquareApp(ctx, client, sandboxEndpoint, id, secret)+ result.ExtraData["Env"] = "Sandbox"+ }++ // verify against prod endpoint+ if verify && !isSandbox(id) {+ isVerified, verificationErr = verifySquareApp(ctx, client, prodEndpoint, id, secret)+ result.ExtraData["Env"] = "Production"}+ result.Verified = isVerified+ result.SetVerificationError(verificationErr)+results = append(results, result)++ // once a secret is verified with id, remove it from the list+ if isVerified {+ delete(uniqueSecretMatches, secret)+ }}}- return++ return results, nil}-func (s Scanner) Type() detectorspb.DetectorType {- return detectorspb.DetectorType_SquareApp+func verifySquareApp(ctx context.Context, client *http.Client, endpoint, id, secret string) (bool, error) {+ reqData, err := json.Marshal(map[string]string{+ "client_id": id,+ "access_token": "fakeTruffleHogAccessTokenForVerification",+ })+ if err != nil {+ return false, err+ }++ req, err := http.NewRequestWithContext(ctx, "POST", endpoint, bytes.NewReader(reqData))+ if err != nil {+ return false, err+ }+ req.Header.Add("Authorization", fmt.Sprintf("Client %s", secret))+ req.Header.Add("Content-Type", "application/json")++ resp, err := client.Do(req)+ if err != nil {+ return false, err+ }++ defer func() {+ _, _ = io.Copy(io.Discard, resp.Body)+ _ = resp.Body.Close()+ }()++ switch resp.StatusCode {+ case http.StatusNotFound:+ return true, nil+ default:+ return false, fmt.Errorf("unexpected status code: %d", resp.StatusCode)+ }}-func (s Scanner) Description() string {- return "Square is a financial services and mobile payment company. Square credentials can be used to access and manage payment processing and other financial services."+func hasSamePrefix(id, secret string) bool {+ idHasPrefix := strings.HasPrefix(id, "sandbox-")+ secretHasPrefix := strings.HasPrefix(secret, "sandbox-")++ return idHasPrefix == secretHasPrefix+}++// isSandbox check if provided key(id or secret) is of sandbox env+func isSandbox(key string) bool {+ return strings.HasPrefix(key, "sandbox-")}diff --git a/pkg/detectors/squareapp/squareapp_integration_test.go b/pkg/detectors/squareapp/squareapp_integration_test.goindex e36008c59..fdec17a3f 100644--- a/pkg/detectors/squareapp/squareapp_integration_test.go+++ b/pkg/detectors/squareapp/squareapp_integration_test.go@@ -19,13 +19,15 @@ import (func TestSquareApp_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")+ testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors5")if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}++ id := testSecrets.MustGetField("SQUAREAPP_ID")secret := testSecrets.MustGetField("SQUAREAPP_SECRET")secretInactive := testSecrets.MustGetField("SQUAREAPP_INACTIVE")- id := testSecrets.MustGetField("SQUAREAPP_ID")+type args struct {ctx context.Contextdata []byte@@ -48,28 +50,23 @@ func TestSquareApp_FromChunk(t *testing.T) {},want: []detectors.Result{{- DetectorType: detectorspb.DetectorType_Square,+ DetectorType: detectorspb.DetectorType_SquareApp,Verified: true,Redacted: id,+ ExtraData: map[string]string{"Env": "Sandbox"},},},wantErr: false,},{- name: "found, unverified",+ name: "found, unverified - detected but not added in result due to mismatch of env",s: Scanner{},args: args{ctx: context.Background(),data: []byte(fmt.Sprintf("You can find a squareapp secret %s within awsId %s", secretInactive, id)),verify: true,},- want: []detectors.Result{- {- DetectorType: detectorspb.DetectorType_Square,- Verified: false,- Redacted: id,- },- },+ want: []detectors.Result{},wantErr: false,},{diff --git a/pkg/detectors/squareapp/squareapp_test.go b/pkg/detectors/squareapp/squareapp_test.goindex 41918ed8c..f39ada19a 100644--- a/pkg/detectors/squareapp/squareapp_test.go+++ b/pkg/detectors/squareapp/squareapp_test.go@@ -12,11 +12,15 @@ import ()var (- validKey = "YJsq0ige-a9khwVJOSwlzBvX0wp4j8t90s2d"- invalidKey = "YJsq0ige-a9kh?VJOSwlzBvX0wp4j8t90s2d"- validSec = "4sSPeeM_jk0VZiTFZJqEwzvXHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"- invalidSec = "4sSPeeM_jk0VZiTFZJqEwz?XHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"- keyword = "squareapp"+ validKey = "sq0ige-a9khwVJOSwlzBvX0wp4j8t90s2d"+ invalidKey = "YJsq0ige-a9kh?VJOSwlzBvX0wp4j8t90s2d"+ validSec = "4sSPeeM_jk0VZiTFZJqEwzvXHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"+ invalidSec = "4sSPeeM_jk0VZiTFZJqEwz?XHjcCd6fsq0cvn-4pvyBIQ1OvY6dOv4X2AK5r6UJaFf0Xkp5NjV6lGhtbM"++ // sandbox+ validSandboxKey = "sandbox-sq0idb-hFAKEQrhLGgFAKELZEDgpo"+ validSandboxSecret = "sandbox-sq0csb-o6cs8xFAKExEgIDGbzn2hFAKEZPbzhe713Q-FAKEfbY"+ keyword = "squareapp")func TestSquareApp_Pattern(t *testing.T) {@@ -32,6 +36,11 @@ func TestSquareApp_Pattern(t *testing.T) {input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, validKey, keyword, validSec),want: []string{validKey},},+ {+ name: "valid sandbox pattern - with keyword squareapp",+ input: fmt.Sprintf("token - '%s'\n secret - '%s'\n", validSandboxKey, validSandboxSecret),+ want: []string{validSandboxKey},+ },{name: "invalid pattern",input: fmt.Sprintf("%s token - '%s'\n%s token - '%s'\n", keyword, invalidKey, keyword, invalidSec),