Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
fix(gcp): handle quoted JSON (#2865)
Richard Gomezc19a46ce13b7
diff --git a/pkg/detectors/gcp/gcp.go b/pkg/detectors/gcp/gcp.goindex 7abcad21a..f8b19edab 100644--- a/pkg/detectors/gcp/gcp.go+++ b/pkg/detectors/gcp/gcp.go@@ -4,6 +4,7 @@ import ("bytes""context""encoding/json"+ "strconv""strings"regexp "github.com/wasilibs/go-re2"@@ -16,10 +17,12 @@ import (type Scanner struct{}// Ensure the Scanner satisfies the interface at compile time.-var _ detectors.Detector = (*Scanner)(nil)-var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)-var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil)-var _ detectors.StartOffsetProvider = (*Scanner)(nil)+var _ interface {+ detectors.Detector+ detectors.CustomFalsePositiveChecker+ detectors.MaxSecretSizeProvider+ detectors.StartOffsetProvider+} = (*Scanner)(nil)var (keyPat = regexp.MustCompile(`\{[^{]+auth_provider_x509_cert_url[^}]+\}`)@@ -38,12 +41,6 @@ type gcpKey struct {ClientX509CertURL string `json:"client_x509_cert_url"`}-func trimCarrots(s string) string {- s = strings.TrimPrefix(s, "<")- s = strings.TrimSuffix(s, ">")- return s-}-// Keywords are used for efficiently pre-filtering chunks.// Use identifiers in the secret preferably, or the provider name.func (s Scanner) Keywords() []string {@@ -60,22 +57,28 @@ const startOffset = 4096// StartOffset returns the start offset for the secret this detector finds.func (Scanner) StartOffset() int64 { return startOffset }+func (s Scanner) Type() detectorspb.DetectorType {+ return detectorspb.DetectorType_GCP+}++func (s Scanner) Description() string {+ return "GCP (Google Cloud Platform) is a suite of cloud computing services that runs on the same infrastructure that Google uses internally for its end-user products. GCP keys can be used to access and manage these services."+}+// FromData will find and optionally verify GCP secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)- matches := keyPat.FindAllString(dataStr, -1)-- for _, match := range matches {- key := match+ uniqueMatches := make(map[string]struct{})+ for _, match := range keyPat.FindAllString(dataStr, -1) {+ uniqueMatches[match] = struct{}{}+ }- key = strings.ReplaceAll(key, `,\\n`, `\n`)- key = strings.ReplaceAll(key, `\"\\n`, `\n`)- key = strings.ReplaceAll(key, `\\"`, `"`)+ for match := range uniqueMatches {+ key := cleanInput(match)creds := gcpKey{}- err := json.Unmarshal([]byte(key), &creds)- if err != nil {+ if err := json.NewDecoder(strings.NewReader(key)).Decode(&creds); err != nil {continue}@@ -83,10 +86,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif strings.Contains(creds.ClientEmail, `<mailto:`) {creds.ClientEmail = strings.Split(strings.Split(creds.ClientEmail, `<mailto:`)[1], `|`)[0]}- creds.AuthProviderX509CertURL = trimCarrots(creds.AuthProviderX509CertURL)- creds.AuthURI = trimCarrots(creds.AuthURI)- creds.ClientX509CertURL = trimCarrots(creds.ClientX509CertURL)- creds.TokenURI = trimCarrots(creds.TokenURI)+ creds.AuthProviderX509CertURL = trimCarets(creds.AuthProviderX509CertURL)+ creds.AuthURI = trimCarets(creds.AuthURI)+ creds.ClientX509CertURL = trimCarets(creds.ClientX509CertURL)+ creds.TokenURI = trimCarets(creds.TokenURI)// Not sure why this might happen, but we've observed this with a verified credraw := []byte(creds.ClientEmail)@@ -106,11 +109,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultRaw: raw,RawV2: credBytes,Redacted: creds.ClientEmail,- }- // Set the RotationGuideURL in the ExtraData- result.ExtraData = map[string]string{- "rotation_guide": "https://howtorotate.com/docs/tutorials/gcp/",- "project": creds.ProjectID,+ ExtraData: map[string]string{+ "rotation_guide": "https://howtorotate.com/docs/tutorials/gcp/",+ "project": creds.ProjectID,+ },}if verify {@@ -118,12 +120,11 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif err != nil {continue}- if credentials != nil {- _, err = credentials.TokenSource.Token()- if err == nil {- result.Verified = true- }++ if _, err = credentials.TokenSource.Token(); err != nil {+ continue}+ result.Verified = true}results = append(results, result)@@ -136,10 +137,28 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {return false, ""}-func (s Scanner) Type() detectorspb.DetectorType {- return detectorspb.DetectorType_GCP+// region Helper methods+func cleanInput(input string) string {+ input = strings.ReplaceAll(input, `,\\n`, `\n`)+ input = strings.ReplaceAll(input, `\"\\n`, `\n`)+ input = strings.ReplaceAll(input, `\\"`, `"`)++ // If the JSON is encoded, it needs to be unquoted for `json.Unmarshal` to succeed.+ // https://github.com/trufflesecurity/trufflehog/issues/2864+ if strings.Contains(input, `\"auth_provider_x509_cert_url\"`) {+ unquoted, err := strconv.Unquote(`"` + input + `"`)+ if err == nil {+ return unquoted+ }+ }++ return input}-func (s Scanner) Description() string {- return "GCP (Google Cloud Platform) is a suite of cloud computing services that runs on the same infrastructure that Google uses internally for its end-user products. GCP keys can be used to access and manage these services."+func trimCarets(s string) string {+ s = strings.TrimPrefix(s, "<")+ s = strings.TrimSuffix(s, ">")+ return s}++//endregiondiff --git a/pkg/detectors/gcp/gcp_test.go b/pkg/detectors/gcp/gcp_test.goindex af5a38e36..3b9d1129f 100644--- a/pkg/detectors/gcp/gcp_test.go+++ b/pkg/detectors/gcp/gcp_test.go@@ -10,8 +10,18 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-var (- validPattern = `[{+func TestGCP_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+ tests := []struct {+ name string+ input string+ want []string+ skip bool+ }{+ {+ name: "valid pattern",+ input: `[{"_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5","name": "GCP","type": "Detector",@@ -35,28 +45,88 @@ var ("expected_response": "200","method": "GET","deprecated": false- }]`- secret = `{"type":"service_account","project_id":"my-test-project","private_key_id":"abc12345def67890ghi","private_key":"-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASC...\n-----END PRIVATE KEY-----\n","client_email":"[email protected]","client_id":"123456789012345678901","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/my-service-account%40my-gcp-project.iam.gserviceaccount.com"}`-)--func TestGCP_Pattern(t *testing.T) {- d := Scanner{}- ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})-- tests := []struct {- name string- input string- want []string- }{+ }]`,+ want: []string{`{"type":"service_account","project_id":"my-test-project","private_key_id":"abc12345def67890ghi","private_key":"-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASC...\n-----END PRIVATE KEY-----\n","client_email":"[email protected]","client_id":"123456789012345678901","auth_uri":"https://accounts.google.com/o/oauth2/auth","token_uri":"https://oauth2.googleapis.com/token","auth_provider_x509_cert_url":"https://www.googleapis.com/oauth2/v1/certs","client_x509_cert_url":"https://www.googleapis.com/robot/v1/metadata/x509/my-service-account%40my-gcp-project.iam.gserviceaccount.com"}`},+ },+ {+ name: "normal file",+ input: `{+ "type": "service_account",+ "project_id": "api-5153635936162123384-123456",+ "private_key_id": "2b387b72ec1b082aa7e52189d9c43f58fb19fb48",+ "private_key": "-----BEGIN PRIVATE KEY-----\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDE/XlaMP419pkUEXAMPLE=\n-----END PRIVATE KEY-----\n",+ "client_email": "unit-test-value@api-5153635936162123384-123456.iam.gserviceaccount.com",+ "client_id": "109763165530299657612",+ "auth_uri": "https://accounts.google.com/o/oauth2/auth",+ "token_uri": "https://oauth2.googleapis.com/token",+ "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs",+ "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/unit-test-value%40api-5153635936162123384-123456.iam.gserviceaccount.com"+}`,+ want: []string{"{\"type\":\"service_account\",\"project_id\":\"api-5153635936162123384-123456\",\"private_key_id\":\"2b387b72ec1b082aa7e52189d9c43f58fb19fb48\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDE/XlaMP419pkUEXAMPLE=\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"unit-test-value@api-5153635936162123384-123456.iam.gserviceaccount.com\",\"client_id\":\"109763165530299657612\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/unit-test-value%40api-5153635936162123384-123456.iam.gserviceaccount.com\"}"},+ },+ {+ name: "escaped JSON",+ input: `{"credentials":"{\n \"type\": \"service_account\",\n \"project_id\": \"unit-test\",\n \"private_key_id\": \"10f922eb17fba903dc59f7baf753976233520012\",\n \"private_key\": \"-----BEGIN PRIVATE KEY-----\\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCgyAZHbtJu1MRf\\ng9+Wg==\\n-----END PRIVATE KEY-----\\n\",\n \"client_email\": \"[email protected]\",\n \"client_id\": \"123456476766156356779\",\n \"auth_uri\": \"https://accounts.google.com/o/oauth2/auth\",\n \"token_uri\": \"https://oauth2.googleapis.com/token\",\n \"auth_provider_x509_cert_url\": \"https://www.googleapis.com/oauth2/v1/certs\",\n \"client_x509_cert_url\": \"https://www.googleapis.com/robot/v1/metadata/x509/fake-value%40unit-test.iam.gserviceaccount.com\"\n}\n"}`,+ want: []string{"{\"type\":\"service_account\",\"project_id\":\"unit-test\",\"private_key_id\":\"10f922eb17fba903dc59f7baf753976233520012\",\"private_key\":\"-----BEGIN PRIVATE KEY-----\\nMIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQCgyAZHbtJu1MRf\\ng9+Wg==\\n-----END PRIVATE KEY-----\\n\",\"client_email\":\"[email protected]\",\"client_id\":\"123456476766156356779\",\"auth_uri\":\"https://accounts.google.com/o/oauth2/auth\",\"token_uri\":\"https://oauth2.googleapis.com/token\",\"auth_provider_x509_cert_url\":\"https://www.googleapis.com/oauth2/v1/certs\",\"client_x509_cert_url\":\"https://www.googleapis.com/robot/v1/metadata/x509/fake-value%40unit-test.iam.gserviceaccount.com\"}g"},+ },+ {+ name: "no private_key_id (1)",+ input: ` {+ resolve: 'gatsby-source-google-spreadsheet',+ options: {+ /// TODO ///+ // Find a better way to include credentials without literally displaying in the config.+ credentials: {+ client_email: "[email protected]",+ private_key: "-----BEGIN PRIVATE KEY-----\nMIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCuT6NTCf5cILhY\nQrqiah7ukgW1yzOR2brT2V1jiux77q22ljeB2XcnVivg97MWLKpPbghvV+zd7QLi\n4WQZL0f7x5W0yTs8CU6cyVNn/dWewEyWQbKifxWoJZiOhxw7cLIoZkY0Vtz9WQy3\nt0rvwMrkRpvA8dSVtgfOr2uSIAxA5vqsvl9VkT+WFy3bU1oKdUcUWGqhB2kg9wJh\nXLFdfGewt1xezEiFom2WTHx1lltQ5NURJALzo9GfelMfeBR2BPlRdrDp+vpPT1V4\nv90JO7V7yaJm5LeyOX1eKmcrQRw/gKnA9XYifdKl01sTIM4p3Rl6mJZe1iPkWVNj\nWGhW+zzZAgMBAAECggEABO85tSXVBsghV9RJwsPEzOWi1j7inpgejxU57NG3uJLs\n3vyZJSqiEiHBG7z/W5sXliiMAhTn3m9xc7lEMjdQfxrrDMNekyhDSrJzU0APDk+s\ny1sgQrPcydYIn85I2RDrZjPg/GXSGzTsZH4Cl89qHvS1v4xJA5Tz2yDBp5ETL+oe\nK3Xba1+TtvsCh14xAnx81yTFIGvjVukh3iXa8PjRdeANrw/WV/0eK6rA5wga20Ik\nsiV8C5t0rXMtF/ZMTagtQa/PL7G890T8CnbulMeYlZ1vhXTlNTMqrL9gl6Scmyuk\n0LWEWeElLHm9C+jDkdPjp+GAF7S6T7QO3r7k4tFHJQKBgQDqjmgtu0nqUqdpl5Un\nr6oUqLCYdMqCl2QjXmBEWdzXqtwo0YeBYGJHI+fc1JhI/XXAZX3AlIvaxzTuEgK6\n3T5pix51jXafFP3mR/WSWj9oFaH3MVp4V28fdnTkZjugPnEDl2vIKYMaiKQdEV0u\nqQLVHi1PT98STp1fnICew3h3/QKBgQC+Pz4qiJUTkuLP1n77K3/GzqVVnQFtPYMr\n1Ob04tDMeAnVL/yW38eP7tVHufeRMGt3A936vl+TF+c2doQLH3zsUFKF368TXsDi\nhSNsxYWeMhi/VejQkpPG7MXZj30nT3AI6YAmcwd5wwWkhGUSDeF6p/H9p89t+f/m\njpyBAB5JDQKBgFjWcxLPGuHLSGkv5mhPmkWU1r4HjiQEHwNeXWvF9WUh65zyLzaL\nQO3c5Za4Vq1egljKl+R23rmQNWXt0GbiIR9sd67iU4lRNBEiNBqoX9eWSfAMG031\nH7t07DUNm4vH2poXodUAFA3arv3rc7WWgeIiOdsOT1jpuaVa60Q2mMwpAoGBAI9d\ng1B0KrtcZpWvE3PdvOWplghlT8ztnOqr/vut7SEimHhSOCvOKUn69jieGMUN0v4W\nKPKrAcUML03ok+r56J8AjJ+cCAg10G8jW6W9V8r1/5Y4fECpJvxzM0mXCv5Tq57b\nr5nJ92k3oQnwR2YKlc9jvkWjbvp2efRZpfDEkQ4FAoGBAJLgC5YaWiFjn2EGtlGa\nKxunQ+zU/TtReRNCA9fjTdlxzH7uqHPrTTZ58TAf8VfI1T01rqY2LPXYsgxfU6ut\nbJrxof0hp00dDKldxFOl5QbvoXhF7T3Bq5RK9uFpxSOpaXvKIMjmYmACgeP1XNNq\n4ThWdWJOMDXLaJN2Mj8n3kih\n-----END PRIVATE KEY-----\n"+ },+ // Spreadsheet is here: https://docs.google.com/spreadsheets/d/...+`,+ skip: true,+ },+ {+ name: "no private_key_id (2)",+ input: `+module.exports = {+ credentials: {+ google: {+ client_email: process.env.GOOGLE_CLIENT_EMAIL || '[email protected]',+ private_key: process.env.GOOGLE_PRIVATE_KEY || '-----BEGIN PRIVATE KEY-----\nYOUR-PRIVATE-KEY-GOES-HERE\n-----END PRIVATE KEY-----\n'+ },`,+ skip: true,+ },{- name: "valid pattern",- input: validPattern,- want: []string{secret},+ name: ".env",+ input: `GOOGLE_SERVICE_ACCOUNT_EMAIL=jedicconnect@f4k3acc.iam.gserviceaccount.com+GOOGLE_SERVICE_ACCOUNT_PRIVATE_KEY="-----BEGIN PRIVATE KEY-----\nMIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCkVEmh423RApn2\nv262VYWKhkyf11erwQPIdQP2f+zSXDxZJwQ2vAY/ecamlYz2TnDoXTM7LghLGXXd\n1ZdcMaAX3dgQdxt36B/gRLOO8eyQwEQjvseSm3NFQjyrX9x94STmrEOWP7xR8DuZ\nhq3hFXT9knD8CW+TwG7rQn0e6mdUqz2tujptLFwo+GUZZ4vbw1Y2pU9QBiVDtA8R\nIVdpDFujLS3nxJEhKUY7WYSWr1cwJwqBNMfIq0GW+Sk8/E25qHztZVsMsvyUtIIF\nH21dGMoyc4CSDWqdgSQvEeyTi9zrlVWA4/gHyb246UlkvEdOBVKH1jiZy76atsLS\nNCtIk6A3AgMBAAECggEAHiluXm19EZj1o4mdi5AE89kUpV4ENH038Yow0QTH9hCB\n7ycvKdC3IN18LcVTWz4okS3SInGfihFBRhdXMc/V/6tzZgpGm2qalzJE9t7Gugbg\nOuNghDNOJA81TYtJ0D0L5d8GhMRsD2oVtmc28RJcJ9LCNDCTLz5p3XqUhTSnBK4h\n2ad5BHgXgCinVXZ+KgCnrm6obijcrlvTGacGXT42UsRA/AJE9Lh2t+zNQE0v6T+B\niI/DEHcq0gNU8yKQSiLsL7P1nc3sLZs7DKsLP0NH1Wd4wgupM9PnPMPAHAwye1tA\nz/hdaUbU94lPW/40rw26DwwX1rfHFavRy/JN2VH52QKBgQDl9v4XxGjgmcted01H\nXPELsfJPod9PCr6M7swUVCRzP2m1CB9efOJBd3yzFI/beQxNjhVH3tUEioHdMgzJ\nrJHB44NyYLaVDFhIcTDlyTKlEYpd7kT5xMGqbcuLRSr0/EH9Ut69+buMqvcXCo9t\nlpgfvx5PizZ6FWCzUeR/qQ5DAwKBgQC27v9UT6dpl47DYUKls1laTsN+wiaFXiuY\n39MPGwXW4AmfY2VqrVuGaGc1+ls9Am75/BkEQr9rDmuy/p+tqeOrILGMskuSzwzO\n5NJ9DbXwgU+rBkRGYP4QdY3eYlsQQJXjrYmtXi03J6NZiVLozMN5Clk6yJLjZue0\nNPMpjl8YXQKBgGfiqmrGObKtB2hHcMu6OtJTsukycRTd/7Le9aaBVG4TyYcUgkdH\nF1cHyXeE5G/7QQmQFCEBky2X/I6WW5yHrtjuFKWI9zJh/0fKipJjz9MuF1nTl6lV\nrz90liz2NC+z/YOY+jLMLGOhoDnydVTGYTaGOgUpGJUSLzsS1ayuDFlNAoGALJRz\nrrU2pBnmFaEHH+BkHwjgxWxE/O1lDH1HLwAz3Rh939TWKzgR/OBGfrYDNAv6xXr1\nEb++bDV4c4dvnF+xdsuh2Rq+JgnFIkpLLWSA+RpaMuB1FP2gDJzJNO+dJ4nFvVVW\nHuS7ehxzx2cayFbWMBIgip3EhhxWzOi2dWMXs5UCgYA6wgiuw7w/Yw+/E6VThyPz\nzAspslQJzT4CzrW1WYwyZPCKAkdxIUd31pVtfj3M8duIpFkT5QidaBY54a3A8aU6\nY3kigasNRpSEMK7/ldMkX9pgPL6LGC1mmb7oTxIJjjAN6xcKVznF756LoK9A7+m6\nNrT7PzoKmZkC7BpmKV4CdQ==\n-----END PRIVATE KEY-----\n"+`,+ skip: true,},+ // TODO: Create an example of these.+ //{+ // name: "Slack mangled email",+ // input: ``,+ // want: []string{""},+ //},+ //{+ // name: "Empty client email",+ // input: ``,+ // want: []string{""},+ //},+ //{+ // name: "Carets",+ // input: ``,+ // want: []string{""},+ //},}for _, test := range tests {t.Run(test.name, func(t *testing.T) {+ if test.skip {+ t.Skip()+ return+ }+matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))if len(matchedDetectors) == 0 {t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)