Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Allow for custom verifier (#1070)
ahrav0052f600905e
diff --git a/main.go b/main.goindex 2fc0fbef7..4b68ddc69 100644--- a/main.go+++ b/main.go@@ -51,6 +51,7 @@ var (printAvgDetectorTime = cli.Flag("print-avg-detector-time", "Print the average time spent on each detector.").Bool()noUpdate = cli.Flag("no-update", "Don't check for updates.").Bool()fail = cli.Flag("fail", "Exit with code 183 if results are found.").Bool()+ verifiers = cli.Flag("verifier", "Set custom verification endpoints.").StringMap()archiveMaxSize = cli.Flag("archive-max-size", "Maximum size of archive to scan. (Byte units eg. 512B, 2KB, 4MB)").Bytes()archiveMaxDepth = cli.Flag("archive-max-depth", "Maximum depth of archive to scan.").Int()archiveTimeout = cli.Flag("archive-timeout", "Maximum time to spend extracting an archive.").Duration()@@ -219,6 +220,8 @@ func run(state overseer.State) {}}+ urls := splitVerifierURLs(*verifiers)+if *archiveMaxSize != 0 {handlers.SetArchiveMaxSize(int(*archiveMaxSize))}@@ -282,6 +285,7 @@ func run(state overseer.State) {engine.WithConcurrency(*concurrency),engine.WithDecoders(decoders.DefaultDecoders()...),engine.WithDetectors(!*noVerification, engine.DefaultDetectors()...),+ engine.WithDetectors(!*noVerification, engine.CustomDetectors(ctx, urls)...),engine.WithDetectors(!*noVerification, conf.Detectors...),engine.WithFilterDetectors(includeFilter),engine.WithFilterDetectors(excludeFilter),@@ -492,6 +496,19 @@ func printAverageDetectorTime(e *engine.Engine) {}}+func splitVerifierURLs(verifierURLs map[string]string) map[string][]string {+ verifiers := make(map[string][]string, len(verifierURLs))+ for k, v := range verifierURLs {+ key := strings.ToLower(k)+ sliceOfValues := strings.Split(v, ",")+ for i, s := range sliceOfValues {+ sliceOfValues[i] = strings.TrimSpace(s)+ }+ verifiers[key] = sliceOfValues+ }+ return verifiers+}+// logFatalFunc returns a log.Fatal style function. Calling the returned// function will terminate the program without cleanup.func logFatalFunc(logger logr.Logger) func(error, string, ...any) {diff --git a/pkg/detectors/github/github.go b/pkg/detectors/github/github.goindex 54abf074f..7b08ab085 100644--- a/pkg/detectors/github/github.go+++ b/pkg/detectors/github/github.go@@ -13,7 +13,34 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/common")-type Scanner struct{}+const defaultURL = "https://api.github.com"++type Scanner struct {+ verifierURLs []string+}++// New creates a new Scanner with the given options.+func New(opts ...func(*Scanner)) *Scanner {+ scanner := &Scanner{+ verifierURLs: make([]string, 0),+ }+ for _, opt := range opts {+ opt(scanner)+ }++ return scanner+}++// WithVerifierURLs adds the given URLs to the list of URLs to check for+// verification of secrets.+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {+ return func(s *Scanner) {+ if includeDefault {+ urls = append(urls, defaultURL)+ }+ s.verifierURLs = append(s.verifierURLs, urls...)+ }+}// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)@@ -29,7 +56,7 @@ var (// https://github.blog/changelog/2022-10-18-introducing-fine-grained-personal-access-tokens/keyPat = regexp.MustCompile(`\b((?:ghp|gho|ghu|ghs|ghr|github_pat)_[a-zA-Z0-9_]{36,255})\b`)- //TODO: Oauth2 client_id and client_secret+ // TODO: Oauth2 client_id and client_secret// https://developer.github.com/v3/#oauth2-keysecret)@@ -71,27 +98,29 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif verify {client := common.SaneHttpClient()// https://developer.github.com/v3/users/#get-the-authenticated-user- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)- if err != nil {- continue- }- req.Header.Add("Content-Type", "application/json; charset=utf-8")- req.Header.Add("Authorization", fmt.Sprintf("token %s", token))- res, err := client.Do(req)- if err == nil {- if res.StatusCode >= 200 && res.StatusCode < 300 {- var userResponse userRes- err = json.NewDecoder(res.Body).Decode(&userResponse)- res.Body.Close()- if err == nil {- s1.Verified = true- s1.ExtraData = map[string]string{- "username": userResponse.Login,- "url": userResponse.UserURL,- "account_type": userResponse.Type,- "site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin),- "name": userResponse.Name,- "company": userResponse.Company,+ for _, url := range s.verifierURLs {+ req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)+ if err != nil {+ continue+ }+ req.Header.Add("Content-Type", "application/json; charset=utf-8")+ req.Header.Add("Authorization", fmt.Sprintf("token %s", token))+ res, err := client.Do(req)+ if err == nil {+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ var userResponse userRes+ err = json.NewDecoder(res.Body).Decode(&userResponse)+ res.Body.Close()+ if err == nil {+ s1.Verified = true+ s1.ExtraData = map[string]string{+ "username": userResponse.Login,+ "url": userResponse.UserURL,+ "account_type": userResponse.Type,+ "site_admin": fmt.Sprintf("%t", userResponse.SiteAdmin),+ "name": userResponse.Name,+ "company": userResponse.Company,+ }}}}diff --git a/pkg/detectors/github_old/github_old.go b/pkg/detectors/github_old/github_old.goindex 2d468d931..f948aa548 100644--- a/pkg/detectors/github_old/github_old.go+++ b/pkg/detectors/github_old/github_old.go@@ -12,7 +12,34 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{}+const defaultURL = "https://api.github.com"++type Scanner struct {+ verifierURLs []string+}++// New creates a new Scanner with the given options.+func New(opts ...func(*Scanner)) *Scanner {+ scanner := &Scanner{+ verifierURLs: make([]string, 0),+ }+ for _, opt := range opts {+ opt(scanner)+ }++ return scanner+}++// WithVerifierURLs adds the given URLs to the list of URLs to check for+// verification of secrets.+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {+ return func(s *Scanner) {+ if includeDefault {+ urls = append(urls, defaultURL)+ }+ s.verifierURLs = append(s.verifierURLs, urls...)+ }+}// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)@@ -63,7 +90,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultcontinue}- s := detectors.Result{+ s1 := detectors.Result{DetectorType: detectorspb.DetectorType_Github,Raw: []byte(token),}@@ -71,30 +98,32 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif verify {client := common.SaneHttpClient()// https://developer.github.com/v3/users/#get-the-authenticated-user- req, err := http.NewRequestWithContext(ctx, "GET", "https://api.github.com/user", nil)- if err != nil {- continue- }- req.Header.Add("Content-Type", "application/json; charset=utf-8")- req.Header.Add("Authorization", fmt.Sprintf("token %s", token))- res, err := client.Do(req)- if err == nil {- if res.StatusCode >= 200 && res.StatusCode < 300 {- var userResponse userRes- err = json.NewDecoder(res.Body).Decode(&userResponse)- res.Body.Close()- if err == nil {- s.Verified = true+ for _, url := range s.verifierURLs {+ req, err := http.NewRequestWithContext(ctx, "GET", fmt.Sprintf("%s/user", url), nil)+ if err != nil {+ continue+ }+ req.Header.Add("Content-Type", "application/json; charset=utf-8")+ req.Header.Add("Authorization", fmt.Sprintf("token %s", token))+ res, err := client.Do(req)+ if err == nil {+ if res.StatusCode >= 200 && res.StatusCode < 300 {+ var userResponse userRes+ err = json.NewDecoder(res.Body).Decode(&userResponse)+ res.Body.Close()+ if err == nil {+ s1.Verified = true+ }}}}}- if !s.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {+ if !s1.Verified && detectors.IsKnownFalsePositive(token, detectors.DefaultFalsePositives, true) {continue}- results = append(results, s)+ results = append(results, s1)}return results, nildiff --git a/pkg/detectors/gitlab/gitlab.go b/pkg/detectors/gitlab/gitlab.goindex 4ed804ec1..bc7990d7a 100644--- a/pkg/detectors/gitlab/gitlab.go+++ b/pkg/detectors/gitlab/gitlab.go@@ -12,7 +12,34 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{}+const defaultURL = "https://gitlab.com"++type Scanner struct {+ verifierURLs []string+}++// New creates a new Scanner with the given options.+func New(opts ...func(*Scanner)) *Scanner {+ scanner := &Scanner{+ verifierURLs: make([]string, 0),+ }+ for _, opt := range opts {+ opt(scanner)+ }++ return scanner+}++// WithVerifierURLs adds the given URLs to the list of URLs to check for+// verification of secrets.+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {+ return func(s *Scanner) {+ if includeDefault {+ urls = append(urls, defaultURL)+ }+ s.verifierURLs = append(s.verifierURLs, urls...)+ }+}// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)@@ -56,24 +83,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result// they all grant access to different parts of the API. I couldn't find an endpoint that every// one of these scopes has access to, so we just check an example endpoint for each scope. If any// of them contain data, we know we have a valid key, but if they all fail, we don't- baseURL := "https://gitlab.com/api/v4"client := common.SaneHttpClient()- // test `read_user` scope- req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil)- if err != nil {- continue- }- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))- res, err := client.Do(req)- if err == nil {- res.Body.Close() // The request body is unused.-- // 200 means good key and has `read_user` scope- // 403 means good key but not the right scope- // 401 is bad key- if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {- secret.Verified = true+ for _, baseURL := range s.verifierURLs {+ // test `read_user` scope+ req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)+ if err != nil {+ continue+ }+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", resMatch))+ res, err := client.Do(req)+ if err == nil {+ res.Body.Close() // The request body is unused.++ // 200 means good key and has `read_user` scope+ // 403 means good key but not the right scope+ // 401 is bad key+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {+ secret.Verified = true+ }}}}diff --git a/pkg/detectors/gitlabv2/gitlab.go b/pkg/detectors/gitlabv2/gitlab.goindex 01ea72228..ffc0b7bf8 100644--- a/pkg/detectors/gitlabv2/gitlab.go+++ b/pkg/detectors/gitlabv2/gitlab.go@@ -11,7 +11,34 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct{}+const defaultURL = "https://gitlab.com"++type Scanner struct {+ verifierURLs []string+}++// New creates a new Scanner with the given options.+func New(opts ...func(*Scanner)) *Scanner {+ scanner := &Scanner{+ verifierURLs: make([]string, 0),+ }+ for _, opt := range opts {+ opt(scanner)+ }++ return scanner+}++// WithVerifierURLs adds the given URLs to the list of URLs to check for+// verification of secrets.+func WithVerifierURLs(urls []string, includeDefault bool) func(*Scanner) {+ return func(s *Scanner) {+ if includeDefault {+ urls = append(urls, defaultURL)+ }+ s.verifierURLs = append(s.verifierURLs, urls...)+ }+}// Ensure the Scanner satisfies the interfaces at compile time.var _ detectors.Detector = (*Scanner)(nil)@@ -50,27 +77,27 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result// they all grant access to different parts of the API. I couldn't find an endpoint that every// one of these scopes has access to, so we just check an example endpoint for each scope. If any// of them contain data, we know we have a valid key, but if they all fail, we don't- baseURL := "https://gitlab.com/api/v4"-client := common.SaneHttpClient()-- // test `read_user` scope- req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/user", nil)- if err != nil {- continue- }- req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))- res, err := client.Do(req)- if err == nil {- res.Body.Close() // The request body is unused.-- // 200 means good key and has `read_user` scope- // 403 means good key but not the right scope- // 401 is bad key- if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {- secret.Verified = true+ for _, baseURL := range s.verifierURLs {+ // test `read_user` scope+ req, err := http.NewRequestWithContext(ctx, "GET", baseURL+"/api/v4/user", nil)+ if err != nil {+ continue+ }+ req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", match[1]))+ res, err := client.Do(req)+ if err == nil {+ res.Body.Close() // The request body is unused.++ // 200 means good key and has `read_user` scope+ // 403 means good key but not the right scope+ // 401 is bad key+ if res.StatusCode == http.StatusOK || res.StatusCode == http.StatusForbidden {+ secret.Verified = true+ }}}+}if !secret.Verified && detectors.IsKnownFalsePositive(string(secret.Raw), detectors.DefaultFalsePositives, true) {diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.goindex 487a99276..6bb3acc2f 100644--- a/pkg/engine/defaults.go+++ b/pkg/engine/defaults.go@@ -1,6 +1,7 @@package engineimport (+ "github.com/trufflesecurity/trufflehog/v3/pkg/context""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abbysale""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/abuseipdb"@@ -731,6 +732,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zipcodebase""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zonkafeedback""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/zulipchat"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/etherscan""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/infura""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/alchemy"@@ -740,6 +742,65 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors/coinmarketcap")+// CustomDetectors returns a list of detectors that are enabled by default, but+// can be overridden by the user.+func CustomDetectors(ctx context.Context, urls map[string][]string) []detectors.Detector {+ defaultDetectors := DefaultDetectors()+ if len(urls) == 0 {+ return defaultDetectors+ }++ for i, detector := range defaultDetectors {++ switch detector.Type() {+ case detectorspb.DetectorType_Github:+ githubUrls, ok := urls["github"]+ if !ok {+ ctx.Logger().V(2).Info("no GitHub urls to ignore")+ continue+ }+ ctx.Logger().V(2).Info("ignoring GitHub urls: %v", githubUrls)++ versioner, ok := detector.(detectors.Versioner)+ if !ok {+ ctx.Logger().V(2).Info("Failed to get version for GitHub detector")+ }+ switch versioner.Version() {+ case 1:+ defaultDetectors[i] = github_old.New(github_old.WithVerifierURLs(githubUrls, true))+ default:+ defaultDetectors[i] = github.New(github.WithVerifierURLs(githubUrls, true))+ }++ case detectorspb.DetectorType_Gitlab:+ gitlabUrls, ok := urls["gitlab"]+ if !ok {+ ctx.Logger().V(2).Info("no GitLab urls to ignore")+ continue+ }+ ctx.Logger().V(2).Info("ignoring GitLab urls: %v", gitlabUrls)++ versioner, ok := detector.(detectors.Versioner)+ if !ok {+ ctx.Logger().V(2).Info("Failed to get version for Gitlab detector")+ }+ switch versioner.Version() {+ case 1:+ defaultDetectors[i] = gitlab.New(gitlab.WithVerifierURLs(gitlabUrls, true))+ default:+ defaultDetectors[i] = gitlabv2.New(gitlabv2.WithVerifierURLs(gitlabUrls, true))+ }++ case detectorspb.DetectorType_JiraToken:+ // TODO(ahrav): Double check that we need to do this.+ default:+ ctx.Logger().V(5).Info("ignoring custom detector", "type", detector.Type())+ continue+ }+ }+ return defaultDetectors+}+func DefaultDetectors() []detectors.Detector {return []detectors.Detector{&heroku.Scanner{},@@ -1505,4 +1566,5 @@ func DefaultDetectors() []detectors.Detector {bscscan.Scanner{},coinmarketcap.Scanner{},}+}diff --git a/pkg/engine/engine.go b/pkg/engine/engine.goindex 2d4eaf9d9..65ff22617 100644--- a/pkg/engine/engine.go+++ b/pkg/engine/engine.go@@ -156,7 +156,7 @@ func Start(ctx context.Context, options ...EngineOption) *Engine {"verification_disabled", len(e.detectors[false]),)- // start the workers+ // Start the workers.for i := 0; i < e.concurrency; i++ {e.workersWg.Add(1)go func() {