Skip to content

Find, verify, and analyze leaked credentials

Custom regex parallel verify (#1127)

ah̳̕mͭͭͨͩ̐e̘ͬ́͋ͬ̊̓͂d2315192fda5f
diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.go
index 27838f9aa..ba1962639 100644
--- a/pkg/custom_detectors/custom_detectors.go
+++ b/pkg/custom_detectors/custom_detectors.go
@@ -12,6 +12,7 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+ "golang.org/x/sync/errgroup"
)
// The maximum number of matches from one chunk. This const is used when
@@ -82,66 +83,97 @@ func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []b
// ]
matches := permutateMatches(regexMatches)
+ g := new(errgroup.Group)
+
// Create result object and test for verification.
+ resultsCh := make(chan detectors.Result, maxTotalMatches)
for _, match := range matches {
+ match := match
+ g.Go(func() error {
+ return c.createResults(ctx, match, verify, resultsCh)
+ })
+ }
+
+ if err := g.Wait(); err != nil {
+ return nil, err
+ }
+ close(resultsCh)
+
+ for result := range resultsCh {
+ results = append(results, result)
+ }
+
+ return results, nil
+}
+
+func (c *customRegexWebhook) createResults(ctx context.Context, match map[string][]string, verify bool, results chan<- detectors.Result) error {
+ if common.IsDone(ctx) {
+ // TODO: Log we're possibly leaving out results.
+ return ctx.Err()
+ }
+ var raw string
+ for _, values := range match {
+ // values[0] contains the entire regex match.
+ raw += values[0]
+ }
+ result := detectors.Result{
+ DetectorType: detectorspb.DetectorType_CustomRegex,
+ Raw: []byte(raw),
+ }
+
+ if !verify {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case results <- result:
+ return nil
+ }
+ }
+ // Verify via webhook.
+ jsonBody, err := json.Marshal(map[string]map[string][]string{
+ c.GetName(): match,
+ })
+ if err != nil {
+ // This should never happen, but if it does, return nil to not
+ // disrupt other verification.
+ return nil
+ }
+ // Try each config until we successfully verify.
+ for _, verifyConfig := range c.GetVerify() {
if common.IsDone(ctx) {
// TODO: Log we're possibly leaving out results.
- return results, nil
- }
- var raw string
- for _, values := range match {
- // values[0] contains the entire regex match.
- raw += values[0]
- }
- result := detectors.Result{
- DetectorType: detectorspb.DetectorType_CustomRegex,
- Raw: []byte(raw),
- }
-
- if !verify {
- results = append(results, result)
- continue
+ return ctx.Err()
}
- // Verify via webhook.
- jsonBody, err := json.Marshal(map[string]map[string][]string{
- c.GetName(): match,
- })
+ req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
if err != nil {
continue
}
- // Try each config until we successfully verify.
- for _, verifyConfig := range c.GetVerify() {
- if common.IsDone(ctx) {
- // TODO: Log we're possibly leaving out results.
- return results, nil
- }
- req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
- if err != nil {
- continue
- }
- for _, header := range verifyConfig.GetHeaders() {
- key, value, found := strings.Cut(header, ":")
- if !found {
- // Should be unreachable due to validation.
- continue
- }
- req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
- }
- res, err := httpClient.Do(req)
- if err != nil {
+ for _, header := range verifyConfig.GetHeaders() {
+ key, value, found := strings.Cut(header, ":")
+ if !found {
+ // Should be unreachable due to validation.
continue
}
- // TODO: Read response body.
- res.Body.Close()
- if res.StatusCode == http.StatusOK {
- result.Verified = true
- break
- }
+ req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
+ }
+ res, err := httpClient.Do(req)
+ if err != nil {
+ continue
+ }
+ // TODO: Read response body.
+ res.Body.Close()
+ if res.StatusCode == http.StatusOK {
+ result.Verified = true
+ break
}
- results = append(results, result)
}
- return results, nil
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case results <- result:
+ return nil
+ }
}
func (c *customRegexWebhook) Keywords() []string {