Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Custom regex parallel verify (#1127)
ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d2315192fda5f
diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.goindex 27838f9aa..ba1962639 100644--- a/pkg/custom_detectors/custom_detectors.go+++ b/pkg/custom_detectors/custom_detectors.go@@ -12,6 +12,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+ "golang.org/x/sync/errgroup")// The maximum number of matches from one chunk. This const is used when@@ -82,66 +83,97 @@ func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []b// ]matches := permutateMatches(regexMatches)+ g := new(errgroup.Group)+// Create result object and test for verification.+ resultsCh := make(chan detectors.Result, maxTotalMatches)for _, match := range matches {+ match := match+ g.Go(func() error {+ return c.createResults(ctx, match, verify, resultsCh)+ })+ }++ if err := g.Wait(); err != nil {+ return nil, err+ }+ close(resultsCh)++ for result := range resultsCh {+ results = append(results, result)+ }++ return results, nil+}++func (c *customRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {+ if common.IsDone(ctx) {+ // TODO: Log we're possibly leaving out results.+ return ctx.Err()+ }+ var raw string+ for _, values := range match {+ // values[0] contains the entire regex match.+ raw += values[0]+ }+ result := detectors.Result{+ DetectorType: detectorspb.DetectorType_CustomRegex,+ Raw: []byte(raw),+ }++ if !verify {+ select {+ case <-ctx.Done():+ return ctx.Err()+ case results <- result:+ return nil+ }+ }+ // Verify via webhook.+ jsonBody, err := json.Marshal(map[string]map[string][]string{+ c.GetName(): match,+ })+ if err != nil {+ // This should never happen, but if it does, return nil to not+ // disrupt other verification.+ return nil+ }+ // Try each config until we successfully verify.+ for _, verifyConfig := range c.GetVerify() {if common.IsDone(ctx) {// TODO: Log we're possibly leaving out results.- return results, nil- }- var raw string- for _, values := range match {- // values[0] contains the entire regex match.- raw += values[0]- }- result := detectors.Result{- DetectorType: detectorspb.DetectorType_CustomRegex,- Raw: []byte(raw),- }-- if !verify {- results = append(results, result)- continue+ return ctx.Err()}- // Verify via webhook.- jsonBody, err := json.Marshal(map[string]map[string][]string{- c.GetName(): match,- })+ req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))if err != nil {continue}- // Try each config until we successfully verify.- for _, verifyConfig := range c.GetVerify() {- if common.IsDone(ctx) {- // TODO: Log we're possibly leaving out results.- return results, nil- }- req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))- if err != nil {- continue- }- for _, header := range verifyConfig.GetHeaders() {- key, value, found := strings.Cut(header, ":")- if !found {- // Should be unreachable due to validation.- continue- }- req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))- }- res, err := httpClient.Do(req)- if err != nil {+ for _, header := range verifyConfig.GetHeaders() {+ key, value, found := strings.Cut(header, ":")+ if !found {+ // Should be unreachable due to validation.continue}- // TODO: Read response body.- res.Body.Close()- if res.StatusCode == http.StatusOK {- result.Verified = true- break- }+ req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))+ }+ res, err := httpClient.Do(req)+ if err != nil {+ continue+ }+ // TODO: Read response body.+ res.Body.Close()+ if res.StatusCode == http.StatusOK {+ result.Verified = true+ break}- results = append(results, result)}- return results, nil+ select {+ case <-ctx.Done():+ return ctx.Err()+ case results <- result:+ return nil+ }}func (c *customRegexWebhook) Keywords() []string {