Skip to content

Find, verify, and analyze leaked credentials

Allow for custom verifier (#1070)

ahrav0052f600905e
diff --git a/main.go b/main.go
index 2fc0fbef7..4b68ddc69 100644
--- a/main.go
+++ b/main.go
@@ -51,6 +51,7 @@ var (
printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()
noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()
fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()
+ verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()
archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()
archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()
archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()
@@ -219,6 +220,8 @@ func run(state overseer.State) {
}
}
+ urls := splitVerifierURLs(*verifiers)
+
if *archiveMaxSize != 0 {
handlers.SetArchiveMaxSize(int(*archiveMaxSize))
}
@@ -282,6 +285,7 @@ func run(state overseer.State) {
engine.WithConcurrency(*concurrency),
engine.WithDecoders(decoders.DefaultDecoders()...),
engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),
+ engine.WithDetectors(!*noVerification, engine.CustomDetectors(ctx, urls)...),
engine.WithDetectors(!*noVerification, conf.Detectors...),
engine.WithFilterDetectors(includeFilter),
engine.WithFilterDetectors(excludeFilter),
@@ -492,6 +496,19 @@ func printAverageDetectorTime(e *engine.Engine) {
}
}
+func splitVerifierURLs(verifierURLs map[string]string) map[string][]string {
+ verifiers := make(map[string][]string, len(verifierURLs))
+ for k, v := range verifierURLs {
+ key := strings.ToLower(k)
+ sliceOfValues := strings.Split(v, ",")
+ for i, s := range sliceOfValues {
+ sliceOfValues[i] = strings.TrimSpace(s)
+ }
+ verifiers[key] = sliceOfValues
+ }
+ return verifiers
+}
+
// logFatalFunc returns a log.Fatal style function. Calling the returned
// function will terminate the program without cleanup.
func logFatalFunc(logger logr.Logger) func(error, string, ...any) {
diff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.go
index 54abf074f..7b08ab085 100644
--- a/pkg/detectors/github/github.go
+++ b/pkg/detectors/github/github.go
@@ -13,7 +13,34 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
)
-type Scanner struct{}
+const defaultURL = "https://api.github.com"
+
+type Scanner struct {
+ verifierURLs []string
+}
+
+// New creates a new Scanner with the given options.
+func New(opts ...func(*Scanner)) *Scanner {
+ scanner := &Scanner{
+ verifierURLs: make([]string, 0),
+ }
+ for _, opt := range opts {
+ opt(scanner)
+ }
+
+ return scanner
+}
+
+// WithVerifierURLs adds the given URLs to the list of URLs to check for
+// verification of secrets.
+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {
+ return func(s *Scanner) {
+ if includeDefault {
+ urls = append(urls, defaultURL)
+ }
+ s.verifierURLs = append(s.verifierURLs, urls...)
+ }
+}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
@@ -29,7 +56,7 @@ var (
// https://github.blog/changelog/2022-10-18-introducing-fine-grained-personal-access-tokens/
keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)
- //TODO: Oauth2 client_id and client_secret
+ // TODO: Oauth2 client_id and client_secret
// https://developer.github.com/v3/#oauth2-keysecret
)
@@ -71,27 +98,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
client := common.SaneHttpClient()
// https://developer.github.com/v3/users/#get-the-authenticated-user
- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
- if err != nil {
- continue
- }
- req.Header.Add("Content-Type", "application/json; charset=utf-8")
- req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
- res, err := client.Do(req)
- if err == nil {
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- var userResponse userRes
- err = json.NewDecoder(res.Body).Decode(&userResponse)
- res.Body.Close()
- if err == nil {
- s1.Verified = true
- s1.ExtraData = map[string]string{
- "username": userResponse.Login,
- "url": userResponse.UserURL,
- "account_type": userResponse.Type,
- "site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin),
- "name": userResponse.Name,
- "company": userResponse.Company,
+ for _, url := range s.verifierURLs {
+ req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)
+ if err != nil {
+ continue
+ }
+ req.Header.Add("Content-Type", "application/json; charset=utf-8")
+ req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
+ res, err := client.Do(req)
+ if err == nil {
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ var userResponse userRes
+ err = json.NewDecoder(res.Body).Decode(&userResponse)
+ res.Body.Close()
+ if err == nil {
+ s1.Verified = true
+ s1.ExtraData = map[string]string{
+ "username": userResponse.Login,
+ "url": userResponse.UserURL,
+ "account_type": userResponse.Type,
+ "site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin),
+ "name": userResponse.Name,
+ "company": userResponse.Company,
+ }
}
}
}
diff --git a/pkg/detectors/github_old/github_old.go b/pkg/detectors/github_old/github_old.go
index 2d468d931..f948aa548 100644
--- a/pkg/detectors/github_old/github_old.go
+++ b/pkg/detectors/github_old/github_old.go
@@ -12,7 +12,34 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{}
+const defaultURL = "https://api.github.com"
+
+type Scanner struct {
+ verifierURLs []string
+}
+
+// New creates a new Scanner with the given options.
+func New(opts ...func(*Scanner)) *Scanner {
+ scanner := &Scanner{
+ verifierURLs: make([]string, 0),
+ }
+ for _, opt := range opts {
+ opt(scanner)
+ }
+
+ return scanner
+}
+
+// WithVerifierURLs adds the given URLs to the list of URLs to check for
+// verification of secrets.
+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {
+ return func(s *Scanner) {
+ if includeDefault {
+ urls = append(urls, defaultURL)
+ }
+ s.verifierURLs = append(s.verifierURLs, urls...)
+ }
+}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
@@ -63,7 +90,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
continue
}
- s := detectors.Result{
+ s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Github,
Raw: []byte(token),
}
@@ -71,30 +98,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if verify {
client := common.SaneHttpClient()
// https://developer.github.com/v3/users/#get-the-authenticated-user
- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)
- if err != nil {
- continue
- }
- req.Header.Add("Content-Type", "application/json; charset=utf-8")
- req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
- res, err := client.Do(req)
- if err == nil {
- if res.StatusCode >= 200 && res.StatusCode < 300 {
- var userResponse userRes
- err = json.NewDecoder(res.Body).Decode(&userResponse)
- res.Body.Close()
- if err == nil {
- s.Verified = true
+ for _, url := range s.verifierURLs {
+ req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)
+ if err != nil {
+ continue
+ }
+ req.Header.Add("Content-Type", "application/json; charset=utf-8")
+ req.Header.Add("Authorization", fmt.Sprintf("token %s", token))
+ res, err := client.Do(req)
+ if err == nil {
+ if res.StatusCode >= 200 && res.StatusCode < 300 {
+ var userResponse userRes
+ err = json.NewDecoder(res.Body).Decode(&userResponse)
+ res.Body.Close()
+ if err == nil {
+ s1.Verified = true
+ }
}
}
}
}
- if !s.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
+ if !s1.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {
continue
}
- results = append(results, s)
+ results = append(results, s1)
}
return results, nil
diff --git a/pkg/detectors/gitlab/gitlab.go b/pkg/detectors/gitlab/gitlab.go
index 4ed804ec1..bc7990d7a 100644
--- a/pkg/detectors/gitlab/gitlab.go
+++ b/pkg/detectors/gitlab/gitlab.go
@@ -12,7 +12,34 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{}
+const defaultURL = "https://gitlab.com"
+
+type Scanner struct {
+ verifierURLs []string
+}
+
+// New creates a new Scanner with the given options.
+func New(opts ...func(*Scanner)) *Scanner {
+ scanner := &Scanner{
+ verifierURLs: make([]string, 0),
+ }
+ for _, opt := range opts {
+ opt(scanner)
+ }
+
+ return scanner
+}
+
+// WithVerifierURLs adds the given URLs to the list of URLs to check for
+// verification of secrets.
+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {
+ return func(s *Scanner) {
+ if includeDefault {
+ urls = append(urls, defaultURL)
+ }
+ s.verifierURLs = append(s.verifierURLs, urls...)
+ }
+}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
@@ -56,24 +83,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// they all grant access to different parts of the API. I couldn't find an endpoint that every
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
// of them contain data, we know we have a valid key, but if they all fail, we don't
- baseURL := "https://gitlab.com/api/v4"
client := common.SaneHttpClient()
- // test `read_user` scope
- req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil)
- if err != nil {
- continue
- }
- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
- res, err := client.Do(req)
- if err == nil {
- res.Body.Close() // The request body is unused.
-
- // 200 means good key and has `read_user` scope
- // 403 means good key but not the right scope
- // 401 is bad key
- if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
- secret.Verified = true
+ for _, baseURL := range s.verifierURLs {
+ // test `read_user` scope
+ req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
+ if err != nil {
+ continue
+ }
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))
+ res, err := client.Do(req)
+ if err == nil {
+ res.Body.Close() // The request body is unused.
+
+ // 200 means good key and has `read_user` scope
+ // 403 means good key but not the right scope
+ // 401 is bad key
+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+ secret.Verified = true
+ }
}
}
}
diff --git a/pkg/detectors/gitlabv2/gitlab.go b/pkg/detectors/gitlabv2/gitlab.go
index 01ea72228..ffc0b7bf8 100644
--- a/pkg/detectors/gitlabv2/gitlab.go
+++ b/pkg/detectors/gitlabv2/gitlab.go
@@ -11,7 +11,34 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct{}
+const defaultURL = "https://gitlab.com"
+
+type Scanner struct {
+ verifierURLs []string
+}
+
+// New creates a new Scanner with the given options.
+func New(opts ...func(*Scanner)) *Scanner {
+ scanner := &Scanner{
+ verifierURLs: make([]string, 0),
+ }
+ for _, opt := range opts {
+ opt(scanner)
+ }
+
+ return scanner
+}
+
+// WithVerifierURLs adds the given URLs to the list of URLs to check for
+// verification of secrets.
+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {
+ return func(s *Scanner) {
+ if includeDefault {
+ urls = append(urls, defaultURL)
+ }
+ s.verifierURLs = append(s.verifierURLs, urls...)
+ }
+}
// Ensure the Scanner satisfies the interfaces at compile time.
var _ detectors.Detector = (*Scanner)(nil)
@@ -50,27 +77,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
// they all grant access to different parts of the API. I couldn't find an endpoint that every
// one of these scopes has access to, so we just check an example endpoint for each scope. If any
// of them contain data, we know we have a valid key, but if they all fail, we don't
- baseURL := "https://gitlab.com/api/v4"
-
client := common.SaneHttpClient()
-
- // test `read_user` scope
- req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil)
- if err != nil {
- continue
- }
- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
- res, err := client.Do(req)
- if err == nil {
- res.Body.Close() // The request body is unused.
-
- // 200 means good key and has `read_user` scope
- // 403 means good key but not the right scope
- // 401 is bad key
- if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
- secret.Verified = true
+ for _, baseURL := range s.verifierURLs {
+ // test `read_user` scope
+ req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)
+ if err != nil {
+ continue
+ }
+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))
+ res, err := client.Do(req)
+ if err == nil {
+ res.Body.Close() // The request body is unused.
+
+ // 200 means good key and has `read_user` scope
+ // 403 means good key but not the right scope
+ // 401 is bad key
+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {
+ secret.Verified = true
+ }
}
}
+
}
if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {
diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.go
index 487a99276..6bb3acc2f 100644
--- a/pkg/engine/defaults.go
+++ b/pkg/engine/defaults.go
@@ -1,6 +1,7 @@
package engine
import (
+ "github.com/trufflesecurity/trufflehog/v3/pkg/context"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"
@@ -731,6 +732,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zipcodebase"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zonkafeedback"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zulipchat"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etherscan"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/infura"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alchemy"
@@ -740,6 +742,65 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinmarketcap"
)
+// CustomDetectors returns a list of detectors that are enabled by default, but
+// can be overridden by the user.
+func CustomDetectors(ctx context.Context, urls map[string][]string) []detectors.Detector {
+ defaultDetectors := DefaultDetectors()
+ if len(urls) == 0 {
+ return defaultDetectors
+ }
+
+ for i, detector := range defaultDetectors {
+
+ switch detector.Type() {
+ case detectorspb.DetectorType_Github:
+ githubUrls, ok := urls["github"]
+ if !ok {
+ ctx.Logger().V(2).Info("no GitHub urls to ignore")
+ continue
+ }
+ ctx.Logger().V(2).Info("ignoring GitHub urls: %v", githubUrls)
+
+ versioner, ok := detector.(detectors.Versioner)
+ if !ok {
+ ctx.Logger().V(2).Info("Failed to get version for GitHub detector")
+ }
+ switch versioner.Version() {
+ case 1:
+ defaultDetectors[i] = github_old.New(github_old.WithVerifierURLs(githubUrls, true))
+ default:
+ defaultDetectors[i] = github.New(github.WithVerifierURLs(githubUrls, true))
+ }
+
+ case detectorspb.DetectorType_Gitlab:
+ gitlabUrls, ok := urls["gitlab"]
+ if !ok {
+ ctx.Logger().V(2).Info("no GitLab urls to ignore")
+ continue
+ }
+ ctx.Logger().V(2).Info("ignoring GitLab urls: %v", gitlabUrls)
+
+ versioner, ok := detector.(detectors.Versioner)
+ if !ok {
+ ctx.Logger().V(2).Info("Failed to get version for Gitlab detector")
+ }
+ switch versioner.Version() {
+ case 1:
+ defaultDetectors[i] = gitlab.New(gitlab.WithVerifierURLs(gitlabUrls, true))
+ default:
+ defaultDetectors[i] = gitlabv2.New(gitlabv2.WithVerifierURLs(gitlabUrls, true))
+ }
+
+ case detectorspb.DetectorType_JiraToken:
+ // TODO(ahrav): Double check that we need to do this.
+ default:
+ ctx.Logger().V(5).Info("ignoring custom detector", "type", detector.Type())
+ continue
+ }
+ }
+ return defaultDetectors
+}
+
func DefaultDetectors() []detectors.Detector {
return []detectors.Detector{
&heroku.Scanner{},
@@ -1505,4 +1566,5 @@ func DefaultDetectors() []detectors.Detector {
bscscan.Scanner{},
coinmarketcap.Scanner{},
}
+
}
diff --git a/pkg/engine/engine.go b/pkg/engine/engine.go
index 2d4eaf9d9..65ff22617 100644
--- a/pkg/engine/engine.go
+++ b/pkg/engine/engine.go
@@ -156,7 +156,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {
"verification_disabled", len(e.detectors[false]),
)
- // start the workers
+ // Start the workers.
for i := 0; i < e.concurrency; i++ {
e.workersWg.Add(1)
go func() {