Skip to content

Find, verify, and analyze leaked credentials

Implement CustomRegex detector (#950)

Miccah861ad057c7a4
diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.go
index c41972896..6a06e0089 100644
--- a/pkg/custom_detectors/custom_detectors.go
+++ b/pkg/custom_detectors/custom_detectors.go
@@ -1,151 +1,226 @@
package custom_detectors
import (
- "fmt"
+ "bytes"
+ "context"
+ "encoding/json"
+ "net/http"
"regexp"
- "strconv"
"strings"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-// customRegex is a CustomRegex that is guaranteed to be valid.
-type customRegex *custom_detectorspb.CustomRegex
+// The maximum number of matches from one chunk. This const is used when
+// permutating each regex match to protect the scanner from doing too much work
+// for poorly defined regexps.
+const maxTotalMatches = 100
-func ValidateKeywords(keywords []string) error {
- if len(keywords) == 0 {
- return fmt.Errorf("no keywords")
- }
-
- for _, keyword := range keywords {
- if len(keyword) == 0 {
- return fmt.Errorf("empty keyword")
- }
- }
- return nil
+// customRegexWebhook is a CustomRegex with webhook validation that is
+// guaranteed to be valid (assuming the data is not changed after
+// initialization).
+type customRegexWebhook struct {
+ *custom_detectorspb.CustomRegex
}
-func ValidateRegex(regex map[string]string) error {
- if len(regex) == 0 {
- return fmt.Errorf("no regex")
- }
-
- for _, r := range regex {
- if _, err := regexp.Compile(r); err != nil {
- return fmt.Errorf("invalid regex %q", r)
- }
- }
-
- return nil
-}
+// Ensure the Scanner satisfies the interface at compile time.
+var _ detectors.Detector = (*customRegexWebhook)(nil)
-func ValidateVerifyEndpoint(endpoint string, unsafe bool) error {
- if len(endpoint) == 0 {
- return fmt.Errorf("no endpoint")
+// NewWebhookCustomRegex initializes and validates a customRegexWebhook. An
+// unexported type is intentionally returned here to ensure the values have
+// been validated.
+func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*customRegexWebhook, error) {
+ // TODO: Return all validation errors.
+ if err := ValidateKeywords(pb.Keywords); err != nil {
+ return nil, err
}
-
- if strings.HasPrefix(endpoint, "http://") && !unsafe {
- return fmt.Errorf("http endpoint must have unsafe=true")
+ if err := ValidateRegex(pb.Regex); err != nil {
+ return nil, err
}
- return nil
-}
-func ValidateVerifyHeaders(headers []string) error {
- for _, header := range headers {
- if !strings.Contains(header, ":") {
- return fmt.Errorf("header %q must contain a colon", header)
+ for _, verify := range pb.Verify {
+ if err := ValidateVerifyEndpoint(verify.Endpoint, verify.Unsafe); err != nil {
+ return nil, err
+ }
+ if err := ValidateVerifyHeaders(verify.Headers); err != nil {
+ return nil, err
}
}
- return nil
-}
-func ValidateVerifyRanges(ranges []string) error {
- const httpLowerBound = 100
- const httpUpperBound = 599
+ // TODO: Copy only necessary data out of pb.
+ return &customRegexWebhook{pb}, nil
+}
- for _, successRange := range ranges {
- if !strings.Contains(successRange, "-") {
- httpCode, err := strconv.Atoi(successRange)
- if err != nil {
- return fmt.Errorf("unable to convert http code to int %q", successRange)
- }
+var httpClient = common.SaneHttpClient()
- if httpCode < httpLowerBound || httpCode > httpUpperBound {
- return fmt.Errorf("invalid http status code %q", successRange)
- }
+func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
+ dataStr := string(data)
+ regexMatches := make(map[string][][]string, len(c.GetRegex()))
+ // Find all submatches for each regex.
+ for name, regex := range c.GetRegex() {
+ regex, err := regexp.Compile(regex)
+ if err != nil {
+ // TODO: Log error.
+ // This should never happen due to validation.
continue
}
+ regexMatches[name] = regex.FindAllStringSubmatch(dataStr, -1)
+ }
- httpRange := strings.Split(successRange, "-")
- if len(httpRange) != 2 {
- return fmt.Errorf("invalid range format %q", successRange)
+ // Permutate each individual match.
+ // {
+ // "foo": [["match1"]]
+ // "bar": [["match2"], ["match3"]]
+ // }
+ // becomes
+ // [
+ // {"foo": ["match1"], "bar": ["match2"]},
+ // {"foo": ["match1"], "bar": ["match3"]},
+ // ]
+ matches := permutateMatches(regexMatches)
+
+ // Create result object and test for verification.
+ for _, match := range matches {
+ if common.IsDone(ctx) {
+ // TODO: Log we're possibly leaving out results.
+ return results, nil
}
-
- lowerBound, err := strconv.Atoi(httpRange[0])
- if err != nil {
- return fmt.Errorf("unable to convert lower bound to int %q", successRange)
+ var raw string
+ for _, values := range match {
+ // values[0] contains the entire regex match.
+ raw += values[0]
}
-
- upperBound, err := strconv.Atoi(httpRange[1])
- if err != nil {
- return fmt.Errorf("unable to convert upper bound to int %q", successRange)
+ result := detectors.Result{
+ DetectorType: detectorspb.DetectorType_CustomRegex,
+ Raw: []byte(raw),
}
- if lowerBound > upperBound {
- return fmt.Errorf("lower bound greater than upper bound on range %q", successRange)
+ if isKnownFalsePositive(match) {
+ continue
}
-
- if lowerBound < httpLowerBound || upperBound > httpUpperBound {
- return fmt.Errorf("invalid http status code range %q", successRange)
+ if !verify {
+ results = append(results, result)
+ continue
}
- }
- return nil
-}
-
-func ValidateRegexVars(regex map[string]string, body ...string) error {
- for _, b := range body {
- matches := NewRegexVarString(b).variables
-
- for match := range matches {
- if _, ok := regex[match]; !ok {
- return fmt.Errorf("body %q contains an unknown variable", b)
+ // Verify via webhook.
+ jsonBody, err := json.Marshal(map[string]map[string][]string{
+ c.GetName(): match,
+ })
+ if err != nil {
+ continue
+ }
+ // Try each config until we successfully verify.
+ for _, verifyConfig := range c.GetVerify() {
+ if common.IsDone(ctx) {
+ // TODO: Log we're possibly leaving out results.
+ return results, nil
+ }
+ req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))
+ if err != nil {
+ continue
+ }
+ for _, header := range verifyConfig.GetHeaders() {
+ key, value, found := strings.Cut(header, ":")
+ if !found {
+ // Should be unreachable due to validation.
+ continue
+ }
+ req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))
+ }
+ res, err := httpClient.Do(req)
+ if err != nil {
+ continue
+ }
+ // TODO: Read response body.
+ res.Body.Close()
+ if res.StatusCode == http.StatusOK {
+ result.Verified = true
+ break
}
}
+ results = append(results, result)
}
- return nil
+ return results, nil
}
-func NewCustomRegex(pb *custom_detectorspb.CustomRegex) (customRegex, error) {
- // TODO: Return all validation errors.
- if err := ValidateKeywords(pb.Keywords); err != nil {
- return nil, err
- }
+func (c *customRegexWebhook) Keywords() []string {
+ return c.GetKeywords()
+}
- if err := ValidateRegex(pb.Regex); err != nil {
- return nil, err
+// productIndices produces a permutation of indices for each length. Example:
+// productIndices(3, 2) -> [[0 0] [1 0] [2 0] [0 1] [1 1] [2 1]]. It returns
+// a slice of length no larger than maxTotalMatches.
+func productIndices(lengths ...int) [][]int {
+ count := 1
+ for _, l := range lengths {
+ count *= l
+ }
+ if count == 0 {
+ return nil
+ }
+ if count > maxTotalMatches {
+ count = maxTotalMatches
}
- for _, verify := range pb.Verify {
-
- if err := ValidateVerifyEndpoint(verify.Endpoint, verify.Unsafe); err != nil {
- return nil, err
+ results := make([][]int, count)
+ for i := 0; i < count; i++ {
+ j := 1
+ result := make([]int, 0, len(lengths))
+ for _, l := range lengths {
+ result = append(result, (i/j)%l)
+ j *= l
}
+ results[i] = result
+ }
+ return results
+}
- if err := ValidateVerifyHeaders(verify.Headers); err != nil {
- return nil, err
- }
+// permutateMatches converts the list of all regex matches into all possible
+// permutations selecting one from each named entry in the map. For example:
+// {"foo": [matchA, matchB], "bar": [matchC]} becomes
+// [{"foo": matchA, "bar": matchC}, {"foo": matchB, "bar": matchC}]
+func permutateMatches(regexMatches map[string][][]string) []map[string][]string {
+ // Get a consistent order for names and their matching lengths.
+ // The lengths are used in calculating the permutation so order matters.
+ names := make([]string, 0, len(regexMatches))
+ lengths := make([]int, 0, len(regexMatches))
+ for key, value := range regexMatches {
+ names = append(names, key)
+ lengths = append(lengths, len(value))
+ }
- if err := ValidateVerifyRanges(verify.SuccessRanges); err != nil {
- return nil, err
+ // Permutate all the indices for each match. For example, if "foo" has
+ // [matchA, matchB] and "bar" has [matchC], we will get indices [0 0] [1 0].
+ permutationIndices := productIndices(lengths...)
+
+ // Build {"foo": matchA, "bar": matchC} and {"foo": matchB, "bar": matchC}
+ // from the indices.
+ var matches []map[string][]string
+ for _, permutation := range permutationIndices {
+ candidate := make(map[string][]string, len(permutationIndices))
+ for i, name := range names {
+ candidate[name] = regexMatches[name][permutation[i]]
}
+ matches = append(matches, candidate)
+ }
- if err := ValidateRegexVars(pb.Regex, append(verify.Headers, verify.Endpoint)...); err != nil {
- return nil, err
- }
+ return matches
+}
+// This function will check false positives for common test words, but also it
+// will make sure the key appears 'random' enough to be a real key.
+func isKnownFalsePositive(match map[string][]string) bool {
+ for _, values := range match {
+ for _, value := range values {
+ if detectors.IsKnownFalsePositive(value, detectors.DefaultFalsePositives, true) {
+ return true
+ }
+ }
}
-
- return pb, nil
+ return false
}
diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.go
index 834f3d3d8..7b4fa85f8 100644
--- a/pkg/custom_detectors/custom_detectors_test.go
+++ b/pkg/custom_detectors/custom_detectors_test.go
@@ -102,226 +102,82 @@ func TestCustomDetectorsParsing(t *testing.T) {
assert.Equal(t, []string{"Authorization: Bearer token"}, got.Verify[0].Headers)
}
-func TestCustomDetectorsKeywordValidation(t *testing.T) {
+func TestProductIndices(t *testing.T) {
tests := []struct {
- name string
- input []string
- wantErr bool
+ name string
+ input []int
+ want [][]int
}{
{
- name: "Test empty list of keywords",
- input: []string{},
- wantErr: true,
+ name: "zero",
+ input: []int{3, 0},
+ want: nil,
},
{
- name: "Test empty keyword",
- input: []string{""},
- wantErr: true,
+ name: "one input",
+ input: []int{3},
+ want: [][]int{{0}, {1}, {2}},
},
{
- name: "Test valid keywords",
- input: []string{"hello", "world"},
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := ValidateKeywords(tt.input)
-
- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
- t.Errorf("ValidateKeywords() error = %v, wantErr %v", got, tt.wantErr)
- }
- })
- }
-}
-
-func TestCustomDetectorsRegexValidation(t *testing.T) {
- tests := []struct {
- name string
- input map[string]string
- wantErr bool
- }{
- {
- name: "Test list of keywords",
- input: map[string]string{
- "id_pat_example": "([a-zA-Z0-9]{32})",
+ name: "two inputs",
+ input: []int{3, 2},
+ want: [][]int{
+ {0, 0}, {1, 0}, {2, 0},
+ {0, 1}, {1, 1}, {2, 1},
},
- wantErr: false,
},
{
- name: "Test empty list of keywords",
- input: map[string]string{},
- wantErr: true,
+ name: "three inputs",
+ input: []int{3, 2, 3},
+ want: [][]int{
+ {0, 0, 0}, {1, 0, 0}, {2, 0, 0},
+ {0, 1, 0}, {1, 1, 0}, {2, 1, 0},
+ {0, 0, 1}, {1, 0, 1}, {2, 0, 1},
+ {0, 1, 1}, {1, 1, 1}, {2, 1, 1},
+ {0, 0, 2}, {1, 0, 2}, {2, 0, 2},
+ {0, 1, 2}, {1, 1, 2}, {2, 1, 2},
+ },
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got := ValidateRegex(tt.input)
-
- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
- t.Errorf("ValidateRegex() error = %v, wantErr %v", got, tt.wantErr)
- }
+ got := productIndices(tt.input...)
+ assert.Equal(t, tt.want, got)
})
}
}
-func TestCustomDetectorsVerifyEndpointValidation(t *testing.T) {
- tests := []struct {
- name string
- endpoint string
- unsafe bool
- wantErr bool
- }{
- {
- name: "Test http endpoint with unsafe flag",
- endpoint: "http://localhost:8000/{id_pat_example}",
- unsafe: true,
- wantErr: false,
- },
- {
- name: "Test http endpoint without unsafe flag",
- endpoint: "http://localhost:8000/{id_pat_example}",
- unsafe: false,
- wantErr: true,
- },
- {
- name: "Test https endpoint with unsafe flag",
- endpoint: "https://localhost:8000/{id_pat_example}",
- unsafe: true,
- wantErr: false,
- },
- {
- name: "Test https endpoint without unsafe flag",
- endpoint: "https://localhost:8000/{id_pat_example}",
- unsafe: false,
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := ValidateVerifyEndpoint(tt.endpoint, tt.unsafe)
-
- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
- t.Errorf("ValidateVerifyEndpoint() error = %v, wantErr %v", got, tt.wantErr)
- }
- })
- }
+func TestProductIndicesMax(t *testing.T) {
+ got := productIndices(2, 3, 4, 5, 6)
+ assert.GreaterOrEqual(t, 2*3*4*5*6, maxTotalMatches)
+ assert.Equal(t, maxTotalMatches, len(got))
}
-func TestCustomDetectorsVerifyHeadersValidation(t *testing.T) {
+func TestPermutateMatches(t *testing.T) {
tests := []struct {
- name string
- headers []string
- wantErr bool
+ name string
+ input map[string][][]string
+ want []map[string][]string
}{
{
- name: "Test single header",
- headers: []string{"Authorization: Bearer {secret_pat_example.0}"},
- wantErr: false,
- },
- {
- name: "Test invalid header",
- headers: []string{"Hello world"},
- wantErr: true,
- },
- {
- name: "Test ugly header",
- headers: []string{"Hello:::::::world::hi:"},
- wantErr: false,
- },
- {
- name: "Test empty header",
- headers: []string{},
- wantErr: false,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := ValidateVerifyHeaders(tt.headers)
-
- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
- t.Errorf("ValidateVerifyHeaders() error = %v, wantErr %v", got, tt.wantErr)
- }
- })
- }
-}
-
-func TestCustomDetectorsVerifyRangeValidation(t *testing.T) {
- tests := []struct {
- name string
- ranges []string
- wantErr bool
- }{
- {
- name: "Test multiple mixed ranges",
- ranges: []string{"200", "300-350"},
- wantErr: false,
- },
- {
- name: "Test invalid non-number range",
- ranges: []string{"hi"},
- wantErr: true,
- },
- {
- name: "Test invalid lower to upper range",
- ranges: []string{"200-100"},
- wantErr: true,
- },
- {
- name: "Test invalid http range",
- ranges: []string{"400-1000"},
- wantErr: true,
- },
- {
- name: "Test multiple ranges with invalid inputs",
- ranges: []string{"322", "hello-world", "100-200"},
- wantErr: true,
+ name: "two matches",
+ input: map[string][][]string{"foo": {{"matchA"}, {"matchB"}}, "bar": {{"matchC"}}},
+ want: []map[string][]string{
+ {"foo": {"matchA"}, "bar": {"matchC"}},
+ {"foo": {"matchB"}, "bar": {"matchC"}},
+ },
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
- got := ValidateVerifyRanges(tt.ranges)
-
- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
- t.Errorf("ValidateVerifyRanges() error = %v, wantErr %v", got, tt.wantErr)
- }
+ got := permutateMatches(tt.input)
+ assert.Equal(t, tt.want, got)
})
}
}
-func TestCustomDetectorsVerifyRegexVarsValidation(t *testing.T) {
- tests := []struct {
- name string
- regex map[string]string
- body string
- wantErr bool
- }{
- {
- name: "Regex defined but not used in body",
- regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
- body: "hello world",
- wantErr: false,
- },
- {
- name: "Regex defined and is used in body",
- regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
- body: "hello world {id}",
- wantErr: false,
- },
- {
- name: "Regex var in body but not defined",
- regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
- body: "hello world {hello}",
- wantErr: true,
- },
- }
- for _, tt := range tests {
- t.Run(tt.name, func(t *testing.T) {
- got := ValidateRegexVars(tt.regex, tt.body)
-
- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
- t.Errorf("ValidateRegexVars() error = %v, wantErr %v", got, tt.wantErr)
- }
- })
+func BenchmarkProductIndices(b *testing.B) {
+ for i := 0; i < b.N; i++ {
+ _ = productIndices(3, 2, 6)
}
}
diff --git a/pkg/custom_detectors/validation.go b/pkg/custom_detectors/validation.go
new file mode 100644
index 000000000..62bc14910
--- /dev/null
+++ b/pkg/custom_detectors/validation.go
@@ -0,0 +1,103 @@
+package custom_detectors
+
+import (
+ "fmt"
+ "strconv"
+ "strings"
+)
+
+func ValidateKeywords(keywords []string) error {
+ if len(keywords) == 0 {
+ return fmt.Errorf("no keywords")
+ }
+
+ for _, keyword := range keywords {
+ if len(keyword) == 0 {
+ return fmt.Errorf("empty keyword")
+ }
+ }
+ return nil
+}
+
+func ValidateRegex(regex map[string]string) error {
+ if len(regex) == 0 {
+ return fmt.Errorf("no regex")
+ }
+ return nil
+}
+
+func ValidateVerifyEndpoint(endpoint string, unsafe bool) error {
+ if len(endpoint) == 0 {
+ return fmt.Errorf("no endpoint")
+ }
+
+ if strings.HasPrefix(endpoint, "http://") && !unsafe {
+ return fmt.Errorf("http endpoint must have unsafe=true")
+ }
+ return nil
+}
+
+func ValidateVerifyHeaders(headers []string) error {
+ for _, header := range headers {
+ if !strings.Contains(header, ":") {
+ return fmt.Errorf("header %q must contain a colon", header)
+ }
+ }
+ return nil
+}
+
+func ValidateVerifyRanges(ranges []string) error {
+ const httpLowerRange = 100
+ const httpUpperRange = 599
+
+ for _, successRange := range ranges {
+ if !strings.Contains(successRange, "-") {
+ httpCode, err := strconv.Atoi(successRange)
+ if err != nil {
+ return fmt.Errorf("unable to convert http code to int %q", successRange)
+ }
+
+ if httpCode < httpLowerRange || httpCode > httpUpperRange {
+ return fmt.Errorf("invalid http status code %q", successRange)
+ }
+
+ continue
+ }
+
+ httpRange := strings.Split(successRange, "-")
+ if len(httpRange) != 2 {
+ return fmt.Errorf("invalid range format %q", successRange)
+ }
+
+ lowerBound, err := strconv.Atoi(httpRange[0])
+ if err != nil {
+ return fmt.Errorf("unable to convert lower bound to int %q", successRange)
+ }
+
+ upperBound, err := strconv.Atoi(httpRange[1])
+ if err != nil {
+ return fmt.Errorf("unable to convert upper bound to int %q", successRange)
+ }
+
+ if lowerBound > upperBound {
+ return fmt.Errorf("lower bound greater than upper bound on range %q", successRange)
+ }
+
+ if lowerBound < httpLowerRange || upperBound > httpUpperRange {
+ return fmt.Errorf("invalid http status code range %q", successRange)
+ }
+ }
+ return nil
+}
+
+func ValidateRegexVars(regex map[string]string, body ...string) error {
+ for _, b := range body {
+ matches := NewRegexVarString(b).variables
+ for match := range matches {
+ if _, ok := regex[match]; !ok {
+ return fmt.Errorf("body %q contains an unknown variable", b)
+ }
+ }
+ }
+ return nil
+}
diff --git a/pkg/custom_detectors/validation_test.go b/pkg/custom_detectors/validation_test.go
new file mode 100644
index 000000000..1aa36f4e0
--- /dev/null
+++ b/pkg/custom_detectors/validation_test.go
@@ -0,0 +1,227 @@
+package custom_detectors
+
+import "testing"
+
+func TestCustomDetectorsKeywordValidation(t *testing.T) {
+ tests := []struct {
+ name string
+ input []string
+ wantErr bool
+ }{
+ {
+ name: "Test empty list of keywords",
+ input: []string{},
+ wantErr: true,
+ },
+ {
+ name: "Test empty keyword",
+ input: []string{""},
+ wantErr: true,
+ },
+ {
+ name: "Test valid keywords",
+ input: []string{"hello", "world"},
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := ValidateKeywords(tt.input)
+
+ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
+ t.Errorf("ValidateKeywords() error = %v, wantErr %v", got, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestCustomDetectorsRegexValidation(t *testing.T) {
+ tests := []struct {
+ name string
+ input map[string]string
+ wantErr bool
+ }{
+ {
+ name: "Test list of keywords",
+ input: map[string]string{
+ "id_pat_example": "([a-zA-Z0-9]{32})",
+ },
+ wantErr: false,
+ },
+ {
+ name: "Test empty list of keywords",
+ input: map[string]string{},
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := ValidateRegex(tt.input)
+
+ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
+ t.Errorf("ValidateRegex() error = %v, wantErr %v", got, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestCustomDetectorsVerifyEndpointValidation(t *testing.T) {
+ tests := []struct {
+ name string
+ endpoint string
+ unsafe bool
+ wantErr bool
+ }{
+ {
+ name: "Test http endpoint with unsafe flag",
+ endpoint: "http://localhost:8000/{id_pat_example}",
+ unsafe: true,
+ wantErr: false,
+ },
+ {
+ name: "Test http endpoint without unsafe flag",
+ endpoint: "http://localhost:8000/{id_pat_example}",
+ unsafe: false,
+ wantErr: true,
+ },
+ {
+ name: "Test https endpoint with unsafe flag",
+ endpoint: "https://localhost:8000/{id_pat_example}",
+ unsafe: true,
+ wantErr: false,
+ },
+ {
+ name: "Test https endpoint without unsafe flag",
+ endpoint: "https://localhost:8000/{id_pat_example}",
+ unsafe: false,
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := ValidateVerifyEndpoint(tt.endpoint, tt.unsafe)
+
+ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
+ t.Errorf("ValidateVerifyEndpoint() error = %v, wantErr %v", got, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestCustomDetectorsVerifyHeadersValidation(t *testing.T) {
+ tests := []struct {
+ name string
+ headers []string
+ wantErr bool
+ }{
+ {
+ name: "Test single header",
+ headers: []string{"Authorization: Bearer {secret_pat_example.0}"},
+ wantErr: false,
+ },
+ {
+ name: "Test invalid header",
+ headers: []string{"Hello world"},
+ wantErr: true,
+ },
+ {
+ name: "Test ugly header",
+ headers: []string{"Hello:::::::world::hi:"},
+ wantErr: false,
+ },
+ {
+ name: "Test empty header",
+ headers: []string{},
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := ValidateVerifyHeaders(tt.headers)
+
+ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
+ t.Errorf("ValidateVerifyHeaders() error = %v, wantErr %v", got, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestCustomDetectorsVerifyRangeValidation(t *testing.T) {
+ tests := []struct {
+ name string
+ ranges []string
+ wantErr bool
+ }{
+ {
+ name: "Test multiple mixed ranges",
+ ranges: []string{"200", "300-350"},
+ wantErr: false,
+ },
+ {
+ name: "Test invalid non-number range",
+ ranges: []string{"hi"},
+ wantErr: true,
+ },
+ {
+ name: "Test invalid lower to upper range",
+ ranges: []string{"200-100"},
+ wantErr: true,
+ },
+ {
+ name: "Test invalid http range",
+ ranges: []string{"400-1000"},
+ wantErr: true,
+ },
+ {
+ name: "Test multiple ranges with invalid inputs",
+ ranges: []string{"322", "hello-world", "100-200"},
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := ValidateVerifyRanges(tt.ranges)
+
+ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
+ t.Errorf("ValidateVerifyRanges() error = %v, wantErr %v", got, tt.wantErr)
+ }
+ })
+ }
+}
+
+func TestCustomDetectorsVerifyRegexVarsValidation(t *testing.T) {
+ tests := []struct {
+ name string
+ regex map[string]string
+ body string
+ wantErr bool
+ }{
+ {
+ name: "Regex defined but not used in body",
+ regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
+ body: "hello world",
+ wantErr: false,
+ },
+ {
+ name: "Regex defined and is used in body",
+ regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
+ body: "hello world {id}",
+ wantErr: false,
+ },
+ {
+ name: "Regex var in body but not defined",
+ regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},
+ body: "hello world {hello}",
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := ValidateRegexVars(tt.regex, tt.body)
+
+ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {
+ t.Errorf("ValidateRegexVars() error = %v, wantErr %v", got, tt.wantErr)
+ }
+ })
+ }
+}
diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.go
index d4012ae06..f81028492 100644
--- a/pkg/pb/detectorspb/detectors.pb.go
+++ b/pkg/pb/detectorspb/detectors.pb.go
@@ -972,6 +972,7 @@ const (
DetectorType_LDAP DetectorType = 901
DetectorType_Shopify DetectorType = 902
DetectorType_RabbitMQ DetectorType = 903
+ DetectorType_CustomRegex DetectorType = 904
)
// Enum value maps for DetectorType.
@@ -1877,6 +1878,7 @@ var (
901: "LDAP",
902: "Shopify",
903: "RabbitMQ",
+ 904: "CustomRegex",
}
DetectorType_value = map[string]int32{
"Alibaba": 0,
@@ -2779,6 +2781,7 @@ var (
"LDAP": 901,
"Shopify": 902,
"RabbitMQ": 903,
+ "CustomRegex": 904,
}
)
@@ -3143,7 +3146,7 @@ var file_detectors_proto_rawDesc = []byte{
0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,
0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,
0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,
- 0x02, 0x2a, 0xff, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
+ 0x02, 0x2a, 0x91, 0x71, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,
0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12,
0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53,
0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a,
@@ -4047,11 +4050,12 @@ var file_detectors_proto_rawDesc = []byte{
0x0a, 0x0a, 0x05, 0x52, 0x65, 0x64, 0x69, 0x73, 0x10, 0x84, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x4c,
0x44, 0x41, 0x50, 0x10, 0x85, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66,
0x79, 0x10, 0x86, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x52, 0x61, 0x62, 0x62, 0x69, 0x74, 0x4d, 0x51,
- 0x10, 0x87, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,
- 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,
- 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,
- 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,
- 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
+ 0x10, 0x87, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67,
+ 0x65, 0x78, 0x10, 0x88, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,
+ 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72,
+ 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76,
+ 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f,
+ 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,
}
var (
diff --git a/proto/detectors.proto b/proto/detectors.proto
index 0b86a59f5..482084c95 100644
--- a/proto/detectors.proto
+++ b/proto/detectors.proto
@@ -911,6 +911,7 @@ enum DetectorType {
LDAP = 901;
Shopify = 902;
RabbitMQ = 903;
+ CustomRegex = 904;
}
message Result {