Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
fix(parseur): ignore false positives (#2229)
Richard Gomez38f36475de31
diff --git a/pkg/detectors/parseur/parseur.go b/pkg/detectors/parseur/parseur.goindex 4cad1da9d..5f389edae 100644--- a/pkg/detectors/parseur/parseur.go+++ b/pkg/detectors/parseur/parseur.go@@ -12,16 +12,18 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{}+type Scanner struct {+ client *http.Client+}// Ensure the Scanner satisfies the interface at compile timevar _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ defaultClient = common.SaneHttpClient()// Make sure that your group is surrounded in boundary characters such as below to reduce false positives- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur"}) + `\b([a-f0-9]{40})\b`)+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur[^il]"}) + `\b([a-f0-9]{40})\b`))// Keywords are used for efficiently pre-filtering chunks.@@ -35,35 +37,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultdataStr := string(data)matches := keyPat.FindAllStringSubmatch(dataStr, -1)-for _, match := range matches {if len(match) != 2 {continue}- resMatch := strings.TrimSpace(match[1])+ resMatch := strings.TrimSpace(match[1])s1 := detectors.Result{DetectorType: detectorspb.DetectorType_Parseur,Raw: []byte(resMatch),}if verify {- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.parseur.com/", nil)- if err != nil {- continue+ if s.client == nil {+ s.client = defaultClient}- req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))- res, err := client.Do(req)- if err == nil {- defer res.Body.Close()- if res.StatusCode >= 200 && res.StatusCode < 300 {- s1.Verified = true- } else {- // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {- continue- }- }+ isVerified, verificationErr := verifyResult(ctx, s.client, resMatch)+ s1.Verified = isVerified+ s1.SetVerificationError(verificationErr, resMatch)+ }++ if !s1.Verified {+ // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {+ continue}}@@ -73,6 +70,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}+func verifyResult(ctx context.Context, client *http.Client, token string) (bool, error) {+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.parseur.com/", nil)+ if err != nil {+ return false, err+ }++ req.Header.Add("Authorization", fmt.Sprintf("Token %s", token))+ res, err := client.Do(req)+ if err != nil {+ return false, err+ }++ defer res.Body.Close()+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ return true, nil+ }+ return false, nil+}+func (s Scanner) Type() detectorspb.DetectorType {return detectorspb.DetectorType_Parseur}diff --git a/pkg/detectors/parseur/parseur_test.go b/pkg/detectors/parseur/parseur_test.goindex fcc9d0c36..9c30cf98d 100644--- a/pkg/detectors/parseur/parseur_test.go+++ b/pkg/detectors/parseur/parseur_test.go@@ -7,12 +7,117 @@ import ("time""github.com/kylelemons/godebug/pretty"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")+func TestParseur_Pattern(t *testing.T) {+ tests := []struct {+ name string+ data string+ shouldMatch bool+ match string+ }{+ // True positives+ {+ name: "valid",+ data: `const parseurToken = "6813a07afc6b4ed35518635c6fb70abf4e721962";`,+ shouldMatch: true,+ match: "6813a07afc6b4ed35518635c6fb70abf4e721962",+ },+ // This technically isn't valid but shouldn't be excluded based on the current pattern.+ {+ name: "valid",+ data: `commit 6813a07afc6b4ed35518635c6fb70abf4e721962+Author: Stéphane Borel <[email protected]>+Date: Thu Dec 30 13:59:59 1999 +0000++ * Modifications de quelques erreurs sur le parseur++commit 2c65bd981d308d264aa0c07083b2bc914905deb3`,+ shouldMatch: true,+ match: "2c65bd981d308d264aa0c07083b2bc914905deb3",+ },++ // False positives+ {+ name: `invalid_parseuri_package.json`,+ data: `{+ "dist": {+ "shasum": "80204a50d4dbb779bfdc6ebe2778d90e4bce320a",+ "tarball": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz"+ },+ "gitHead": "792c9a63162a4484eb6b4f95fc611ccf224a24b6",`,+ shouldMatch: false,+ },+ // https://github.com/airalab/airapkgs/blob/cb3f8021303f79345f65b5328b75117044bde852/pkgs/servers/mesh/meshviewer/yarn.nix#L6066+ {+ name: `invalid_parseuri_nix`,+ data: `+ {+ name = "parseuri-0.0.5.tgz";+ path = fetchurl {+ name = "parseuri-0.0.5.tgz";+ url = "https://registry.yarnpkg.com/parseuri/-/parseuri-0.0.5.tgz";+ sha1 = "80204a50d4dbb779bfdc6ebe2778d90e4bce320a";+ };+ }`,+ shouldMatch: false,+ },+ {+ name: `invalid_parseurl_yarn`,+ data: `parseurl@~1.3.1:+ version "1.3.1"+ resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56"`,+ shouldMatch: false,+ },+ // https://github.com/tolerious/django-wechat/blob/18f3f2d5d8377c7dde8700afc5977861c8488b68/django_weixin/Sample.py#L30+ {+ name: `invalid_parseurl_func`,+ data: `#sVerifyMsgSig=HttpUtils.ParseUrl("msg_signature")+ sVerifyMsgSig="5c45ff5e21c57e6ad56bac8758b79b1d9ac89fd3"`,+ shouldMatch: false,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ s := Scanner{}++ results, err := s.FromData(context.Background(), false, []byte(test.data))+ if err != nil {+ t.Errorf("Parseur.FromData() error = %v", err)+ return+ }++ if test.shouldMatch {+ if len(results) == 0 {+ t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)+ return+ }+ expected := test.data+ if test.match != "" {+ expected = test.match+ }+ result := results[0]+ resultData := string(result.Raw)+ if resultData != expected {+ t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)+ return+ }+ } else {+ if len(results) > 0 {+ t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)+ return+ }+ }+ })+ }+}+func TestParseur_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()