Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Ignore known common prefix matches for Github V1 detector (#4379)
Kashif Khan6dd65a515659
diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.goindex 337b21ffa..4165c19f7 100644--- a/pkg/detectors/github/v1/github_old.go+++ b/pkg/detectors/github/v1/github_old.go@@ -28,8 +28,8 @@ func (Scanner) CloudEndpoint() string { return "https://api.github.com" }var (// Oauth token// https://developer.github.com/v3/#oauth2-token-sent-in-a-header- // the middle regex `\b[a-zA-Z0-9.\/?=&]{0,40}` is to match the prefix of token match to avoid processing common known patterns- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b[a-zA-Z0-9.\/?=&]{0,40}` + `\b([a-f0-9]{40})\b`)+ // the middle regex `(?:[a-zA-Z0-9.\/?=&:-]{0,40})` is to match the prefix of token match to avoid processing common known patterns+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b(?:[a-zA-Z0-9.\/?=&:-]{0,40})([a-f0-9]{40})\b`)// TODO: Oauth2 client_id and client_secret// https://developer.github.com/v3/#oauth2-keysecret@@ -73,7 +73,15 @@ var ghFalsePositives = map[detectors.FalsePositive]struct{}{}var ghKnownNonSensitivePrefixes = []string{- "avatars.githubusercontent.com", // github avatar urls prefix+ "avatars.githubusercontent.com", // GitHub avatar URLs+ "actions/", // GitHub Actions paths+ "raw.githubusercontent.com/", // Raw file URLs from GitHub+ "api.github.com/repos/", // GitHub API repository endpoints+ "gist.github.com/", // GitHub Gist URLs+ "sha256:", // SHA256 hash prefix+ "github.com/", // General GitHub repo URLs+ "pipelines.actions.githubusercontent.com/", // GitHub Actions infrastructure+ "ghcr.io/", // GitHub Container Registry}// FromData will find and optionally verify GitHub secrets in a given set of bytes.@@ -93,11 +101,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultcontinue}- // to avoid false positives- if isKnownNonSensitiveCommonPrefix(matchPrefix) {- continue- }-if detectors.StringShannonEntropy(token) < 3.5 {continue}@@ -116,6 +119,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultclient := common.SaneHttpClient()isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)++ if !isVerified {+ // to avoid false positives for unverified findings+ if isKnownNonSensitiveCommonPrefix(matchPrefix) {+ continue+ }+ }+s1.Verified = isVerifieds1.SetVerificationError(err, token)diff --git a/pkg/detectors/github/v1/github_old_test.go b/pkg/detectors/github/v1/github_old_test.goindex f4a4eea48..d3a319ab7 100644--- a/pkg/detectors/github/v1/github_old_test.go+++ b/pkg/detectors/github/v1/github_old_test.go@@ -5,29 +5,10 @@ import ("testing""github.com/google/go-cmp/cmp"-"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")-var (- validPattern = `[{- "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",- "name": "Github",- "type": "Detector",- "api": true,- "authentication_type": "",- "verification_url": "https://api.example.com/example",- "test_secrets": {- "github_secret": "abc123def4567890abcdef1234567890abcdef12"- },- "expected_response": "200",- "method": "GET",- "deprecated": false- }]`- secret = "abc123def4567890abcdef1234567890abcdef12"-)-func TestGithub_Pattern(t *testing.T) {d := Scanner{}ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})@@ -38,9 +19,22 @@ func TestGithub_Pattern(t *testing.T) {want []string}{{- name: "valid pattern",- input: validPattern,- want: []string{secret},+ name: "valid pattern",+ input: `[{+ "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",+ "name": "Github",+ "type": "Detector",+ "api": true,+ "authentication_type": "",+ "verification_url": "https://api.example.com/example",+ "test_secrets": {+ "github_secret": "abc123def4567890abcdef1234567890abcdef12"+ },+ "expected_response": "200",+ "method": "GET",+ "deprecated": false+ }]`,+ want: []string{"abc123def4567890abcdef1234567890abcdef12"},},}@@ -86,3 +80,42 @@ func TestGithub_Pattern(t *testing.T) {})}}++func Test_isKnownNonSensitiveCommonPrefix(t *testing.T) {+ type args struct {+ matchPrefix string+ }+ tests := []struct {+ name string+ args args+ isKnownPrefix bool+ }{+ {+ name: "repo url",+ args: args{matchPrefix: "github.com/repos/symfony/monolog-bridge/zipball/9d14621e59f22c2b6d030d92d37ffe5ae1e60452"},+ isKnownPrefix: true,+ },+ {+ name: "sha256 hash",+ args: args{matchPrefix: "Digest: sha256:f9a92af4d46ca171bffa5c00509414a19d9887c9ed4fe98d1f43757b52600e39"},+ isKnownPrefix: true,+ },+ {+ name: "real looking token",+ args: args{matchPrefix: "github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e"},+ isKnownPrefix: false,+ },+ {+ name: "github url",+ args: args{matchPrefix: "github.com/wrandelshofer/FastDoubleParser/blob/39e123b15b71f29a38a087d16a0bc620fc879aa6/bigint-LICENSE"},+ isKnownPrefix: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ if got := isKnownNonSensitiveCommonPrefix(tt.args.matchPrefix); got != tt.isKnownPrefix {+ t.Errorf("isKnownNonSensitiveCommonPrefix() = %v, want %v", got, tt.isKnownPrefix)+ }+ })+ }+}