Skip to content

Find, verify, and analyze leaked credentials

Ignore known common prefix matches for Github V1 detector (#4379)

Kashif Khan6dd65a515659
diff --git a/pkg/detectors/github/v1/github_old.go b/pkg/detectors/github/v1/github_old.go
index 337b21ffa..4165c19f7 100644
--- a/pkg/detectors/github/v1/github_old.go
+++ b/pkg/detectors/github/v1/github_old.go
@@ -28,8 +28,8 @@ func (Scanner) CloudEndpoint() string { return "https://api.github.com" }
var (
// Oauth token
// https://developer.github.com/v3/#oauth2-token-sent-in-a-header
- // the middle regex `\b[a-zA-Z0-9.\/?=&]{0,40}` is to match the prefix of token match to avoid processing common known patterns
- keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b[a-zA-Z0-9.\/?=&]{0,40}` + `\b([a-f0-9]{40})\b`)
+ // the middle regex `(?:[a-zA-Z0-9.\/?=&:-]{0,40})` is to match the prefix of token match to avoid processing common known patterns
+ keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"github", "gh", "pat", "token"}) + `\b(?:[a-zA-Z0-9.\/?=&:-]{0,40})([a-f0-9]{40})\b`)
// TODO: Oauth2 client_id and client_secret
// https://developer.github.com/v3/#oauth2-keysecret
@@ -73,7 +73,15 @@ var ghFalsePositives = map[detectors.FalsePositive]struct{}{
}
var ghKnownNonSensitivePrefixes = []string{
- "avatars.githubusercontent.com", // github avatar urls prefix
+ "avatars.githubusercontent.com", // GitHub avatar URLs
+ "actions/", // GitHub Actions paths
+ "raw.githubusercontent.com/", // Raw file URLs from GitHub
+ "api.github.com/repos/", // GitHub API repository endpoints
+ "gist.github.com/", // GitHub Gist URLs
+ "sha256:", // SHA256 hash prefix
+ "github.com/", // General GitHub repo URLs
+ "pipelines.actions.githubusercontent.com/", // GitHub Actions infrastructure
+ "ghcr.io/", // GitHub Container Registry
}
// FromData will find and optionally verify GitHub secrets in a given set of bytes.
@@ -93,11 +101,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
- // to avoid false positives
- if isKnownNonSensitiveCommonPrefix(matchPrefix) {
- continue
- }
-
if detectors.StringShannonEntropy(token) < 3.5 {
continue
}
@@ -116,6 +119,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
client := common.SaneHttpClient()
isVerified, userResponse, headers, err := s.VerifyGithub(ctx, client, token)
+
+ if !isVerified {
+ // to avoid false positives for unverified findings
+ if isKnownNonSensitiveCommonPrefix(matchPrefix) {
+ continue
+ }
+ }
+
s1.Verified = isVerified
s1.SetVerificationError(err, token)
diff --git a/pkg/detectors/github/v1/github_old_test.go b/pkg/detectors/github/v1/github_old_test.go
index f4a4eea48..d3a319ab7 100644
--- a/pkg/detectors/github/v1/github_old_test.go
+++ b/pkg/detectors/github/v1/github_old_test.go
@@ -5,29 +5,10 @@ import (
"testing"
"github.com/google/go-cmp/cmp"
-
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
-var (
- validPattern = `[{
- "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",
- "name": "Github",
- "type": "Detector",
- "api": true,
- "authentication_type": "",
- "verification_url": "https://api.example.com/example",
- "test_secrets": {
- "github_secret": "abc123def4567890abcdef1234567890abcdef12"
- },
- "expected_response": "200",
- "method": "GET",
- "deprecated": false
- }]`
- secret = "abc123def4567890abcdef1234567890abcdef12"
-)
-
func TestGithub_Pattern(t *testing.T) {
d := Scanner{}
ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
@@ -38,9 +19,22 @@ func TestGithub_Pattern(t *testing.T) {
want []string
}{
{
- name: "valid pattern",
- input: validPattern,
- want: []string{secret},
+ name: "valid pattern",
+ input: `[{
+ "_id": "1a8d0cca-e1a9-4318-bc2f-f5658ab2dcb5",
+ "name": "Github",
+ "type": "Detector",
+ "api": true,
+ "authentication_type": "",
+ "verification_url": "https://api.example.com/example",
+ "test_secrets": {
+ "github_secret": "abc123def4567890abcdef1234567890abcdef12"
+ },
+ "expected_response": "200",
+ "method": "GET",
+ "deprecated": false
+ }]`,
+ want: []string{"abc123def4567890abcdef1234567890abcdef12"},
},
}
@@ -86,3 +80,42 @@ func TestGithub_Pattern(t *testing.T) {
})
}
}
+
+func Test_isKnownNonSensitiveCommonPrefix(t *testing.T) {
+ type args struct {
+ matchPrefix string
+ }
+ tests := []struct {
+ name string
+ args args
+ isKnownPrefix bool
+ }{
+ {
+ name: "repo url",
+ args: args{matchPrefix: "github.com/repos/symfony/monolog-bridge/zipball/9d14621e59f22c2b6d030d92d37ffe5ae1e60452"},
+ isKnownPrefix: true,
+ },
+ {
+ name: "sha256 hash",
+ args: args{matchPrefix: "Digest: sha256:f9a92af4d46ca171bffa5c00509414a19d9887c9ed4fe98d1f43757b52600e39"},
+ isKnownPrefix: true,
+ },
+ {
+ name: "real looking token",
+ args: args{matchPrefix: "github-app-token@df432ceedc7162793a195dd1713ff69aefc7379e"},
+ isKnownPrefix: false,
+ },
+ {
+ name: "github url",
+ args: args{matchPrefix: "github.com/wrandelshofer/FastDoubleParser/blob/39e123b15b71f29a38a087d16a0bc620fc879aa6/bigint-LICENSE"},
+ isKnownPrefix: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ if got := isKnownNonSensitiveCommonPrefix(tt.args.matchPrefix); got != tt.isKnownPrefix {
+ t.Errorf("isKnownNonSensitiveCommonPrefix() = %v, want %v", got, tt.isKnownPrefix)
+ }
+ })
+ }
+}