Skip to content

Find, verify, and analyze leaked credentials

fix(parseur): ignore false positives (#2229)

Richard Gomez38f36475de31
diff --git a/pkg/detectors/parseur/parseur.go b/pkg/detectors/parseur/parseur.go
index 4cad1da9d..5f389edae 100644
--- a/pkg/detectors/parseur/parseur.go
+++ b/pkg/detectors/parseur/parseur.go
@@ -12,16 +12,18 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{}
+type Scanner struct {
+ client *http.Client
+}
// Ensure the Scanner satisfies the interface at compile time
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ defaultClient = common.SaneHttpClient()
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur"}) + `\b([a-f0-9]{40})\b`)
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"parseur[^il]"}) + `\b([a-f0-9]{40})\b`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -35,35 +37,30 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
dataStr := string(data)
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
-
for _, match := range matches {
if len(match) != 2 {
continue
}
- resMatch := strings.TrimSpace(match[1])
+ resMatch := strings.TrimSpace(match[1])
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Parseur,
Raw: []byte(resMatch),
}
if verify {
- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.parseur.com/", nil)
- if err != nil {
- continue
+ if s.client == nil {
+ s.client = defaultClient
}
- req.Header.Add("Authorization", fmt.Sprintf("Token %s", resMatch))
- res, err := client.Do(req)
- if err == nil {
- defer res.Body.Close()
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- s1.Verified = true
- } else {
- // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
- if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
- continue
- }
- }
+ isVerified, verificationErr := verifyResult(ctx, s.client, resMatch)
+ s1.Verified = isVerified
+ s1.SetVerificationError(verificationErr, resMatch)
+ }
+
+ if !s1.Verified {
+ // This function will check false positives for common test words, but also it will make sure the key appears 'random' enough to be a real key
+ if detectors.IsKnownFalsePositive(resMatch, detectors.DefaultFalsePositives, true) {
+ continue
}
}
@@ -73,6 +70,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
+func verifyResult(ctx context.Context, client *http.Client, token string) (bool, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, "https://api.parseur.com/", nil)
+ if err != nil {
+ return false, err
+ }
+
+ req.Header.Add("Authorization", fmt.Sprintf("Token %s", token))
+ res, err := client.Do(req)
+ if err != nil {
+ return false, err
+ }
+
+ defer res.Body.Close()
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ return true, nil
+ }
+ return false, nil
+}
+
func (s Scanner) Type() detectorspb.DetectorType {
return detectorspb.DetectorType_Parseur
}
diff --git a/pkg/detectors/parseur/parseur_test.go b/pkg/detectors/parseur/parseur_test.go
index fcc9d0c36..9c30cf98d 100644
--- a/pkg/detectors/parseur/parseur_test.go
+++ b/pkg/detectors/parseur/parseur_test.go
@@ -7,12 +7,117 @@ import (
"time"
"github.com/kylelemons/godebug/pretty"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
+func TestParseur_Pattern(t *testing.T) {
+ tests := []struct {
+ name string
+ data string
+ shouldMatch bool
+ match string
+ }{
+ // True positives
+ {
+ name: "valid",
+ data: `const parseurToken = "6813a07afc6b4ed35518635c6fb70abf4e721962";`,
+ shouldMatch: true,
+ match: "6813a07afc6b4ed35518635c6fb70abf4e721962",
+ },
+ // This technically isn't valid but shouldn't be excluded based on the current pattern.
+ {
+ name: "valid",
+ data: `commit 6813a07afc6b4ed35518635c6fb70abf4e721962
+Author: Stéphane Borel <[email protected]>
+Date: Thu Dec 30 13:59:59 1999 +0000
+
+ * Modifications de quelques erreurs sur le parseur
+
+commit 2c65bd981d308d264aa0c07083b2bc914905deb3`,
+ shouldMatch: true,
+ match: "2c65bd981d308d264aa0c07083b2bc914905deb3",
+ },
+
+ // False positives
+ {
+ name: `invalid_parseuri_package.json`,
+ data: `{
+ "dist": {
+ "shasum": "80204a50d4dbb779bfdc6ebe2778d90e4bce320a",
+ "tarball": "https://registry.npmjs.org/parseuri/-/parseuri-0.0.5.tgz"
+ },
+ "gitHead": "792c9a63162a4484eb6b4f95fc611ccf224a24b6",`,
+ shouldMatch: false,
+ },
+ // https://github.com/airalab/airapkgs/blob/cb3f8021303f79345f65b5328b75117044bde852/pkgs/servers/mesh/meshviewer/yarn.nix#L6066
+ {
+ name: `invalid_parseuri_nix`,
+ data: `
+ {
+ name = "parseuri-0.0.5.tgz";
+ path = fetchurl {
+ name = "parseuri-0.0.5.tgz";
+ url = "https://registry.yarnpkg.com/parseuri/-/parseuri-0.0.5.tgz";
+ sha1 = "80204a50d4dbb779bfdc6ebe2778d90e4bce320a";
+ };
+ }`,
+ shouldMatch: false,
+ },
+ {
+ name: `invalid_parseurl_yarn`,
+ data: `parseurl@~1.3.1:
+ version "1.3.1"
+ resolved "https://registry.yarnpkg.com/parseurl/-/parseurl-1.3.1.tgz#c8ab8c9223ba34888aa64a297b28853bec18da56"`,
+ shouldMatch: false,
+ },
+ // https://github.com/tolerious/django-wechat/blob/18f3f2d5d8377c7dde8700afc5977861c8488b68/django_weixin/Sample.py#L30
+ {
+ name: `invalid_parseurl_func`,
+ data: `#sVerifyMsgSig=HttpUtils.ParseUrl("msg_signature")
+ sVerifyMsgSig="5c45ff5e21c57e6ad56bac8758b79b1d9ac89fd3"`,
+ shouldMatch: false,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := Scanner{}
+
+ results, err := s.FromData(context.Background(), false, []byte(test.data))
+ if err != nil {
+ t.Errorf("Parseur.FromData() error = %v", err)
+ return
+ }
+
+ if test.shouldMatch {
+ if len(results) == 0 {
+ t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
+ return
+ }
+ expected := test.data
+ if test.match != "" {
+ expected = test.match
+ }
+ result := results[0]
+ resultData := string(result.Raw)
+ if resultData != expected {
+ t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, resultData)
+ return
+ }
+ } else {
+ if len(results) > 0 {
+ t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
+ return
+ }
+ }
+ })
+ }
+}
+
func TestParseur_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()