Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Implement CustomRegex detector (#950)
Miccah861ad057c7a4
diff --git a/pkg/custom_detectors/custom_detectors.go b/pkg/custom_detectors/custom_detectors.goindex c41972896..6a06e0089 100644--- a/pkg/custom_detectors/custom_detectors.go+++ b/pkg/custom_detectors/custom_detectors.go@@ -1,151 +1,226 @@package custom_detectorsimport (- "fmt"+ "bytes"+ "context"+ "encoding/json"+ "net/http""regexp"- "strconv""strings"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/custom_detectorspb"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-// customRegex is a CustomRegex that is guaranteed to be valid.-type customRegex *custom_detectorspb.CustomRegex+// The maximum number of matches from one chunk. This const is used when+// permutating each regex match to protect the scanner from doing too much work+// for poorly defined regexps.+const maxTotalMatches = 100-func ValidateKeywords(keywords []string) error {- if len(keywords) == 0 {- return fmt.Errorf("no keywords")- }-- for _, keyword := range keywords {- if len(keyword) == 0 {- return fmt.Errorf("empty keyword")- }- }- return nil+// customRegexWebhook is a CustomRegex with webhook validation that is+// guaranteed to be valid (assuming the data is not changed after+// initialization).+type customRegexWebhook struct {+ *custom_detectorspb.CustomRegex}-func ValidateRegex(regex map[string]string) error {- if len(regex) == 0 {- return fmt.Errorf("no regex")- }-- for _, r := range regex {- if _, err := regexp.Compile(r); err != nil {- return fmt.Errorf("invalid regex %q", r)- }- }-- return nil-}+// Ensure the Scanner satisfies the interface at compile time.+var _ detectors.Detector = (*customRegexWebhook)(nil)-func ValidateVerifyEndpoint(endpoint string, unsafe bool) error {- if len(endpoint) == 0 {- return fmt.Errorf("no endpoint")+// NewWebhookCustomRegex initializes and validates a customRegexWebhook. An+// unexported type is intentionally returned here to ensure the values have+// been validated.+func NewWebhookCustomRegex(pb *custom_detectorspb.CustomRegex) (*customRegexWebhook, error) {+ // TODO: Return all validation errors.+ if err := ValidateKeywords(pb.Keywords); err != nil {+ return nil, err}-- if strings.HasPrefix(endpoint, "http://") && !unsafe {- return fmt.Errorf("http endpoint must have unsafe=true")+ if err := ValidateRegex(pb.Regex); err != nil {+ return nil, err}- return nil-}-func ValidateVerifyHeaders(headers []string) error {- for _, header := range headers {- if !strings.Contains(header, ":") {- return fmt.Errorf("header %q must contain a colon", header)+ for _, verify := range pb.Verify {+ if err := ValidateVerifyEndpoint(verify.Endpoint, verify.Unsafe); err != nil {+ return nil, err+ }+ if err := ValidateVerifyHeaders(verify.Headers); err != nil {+ return nil, err}}- return nil-}-func ValidateVerifyRanges(ranges []string) error {- const httpLowerBound = 100- const httpUpperBound = 599+ // TODO: Copy only necessary data out of pb.+ return &customRegexWebhook{pb}, nil+}- for _, successRange := range ranges {- if !strings.Contains(successRange, "-") {- httpCode, err := strconv.Atoi(successRange)- if err != nil {- return fmt.Errorf("unable to convert http code to int %q", successRange)- }+var httpClient = common.SaneHttpClient()- if httpCode < httpLowerBound || httpCode > httpUpperBound {- return fmt.Errorf("invalid http status code %q", successRange)- }+func (c *customRegexWebhook) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)+ regexMatches := make(map[string][][]string, len(c.GetRegex()))+ // Find all submatches for each regex.+ for name, regex := range c.GetRegex() {+ regex, err := regexp.Compile(regex)+ if err != nil {+ // TODO: Log error.+ // This should never happen due to validation.continue}+ regexMatches[name] = regex.FindAllStringSubmatch(dataStr, -1)+ }- httpRange := strings.Split(successRange, "-")- if len(httpRange) != 2 {- return fmt.Errorf("invalid range format %q", successRange)+ // Permutate each individual match.+ // {+ // "foo": [["match1"]]+ // "bar": [["match2"], ["match3"]]+ // }+ // becomes+ // [+ // {"foo": ["match1"], "bar": ["match2"]},+ // {"foo": ["match1"], "bar": ["match3"]},+ // ]+ matches := permutateMatches(regexMatches)++ // Create result object and test for verification.+ for _, match := range matches {+ if common.IsDone(ctx) {+ // TODO: Log we're possibly leaving out results.+ return results, nil}-- lowerBound, err := strconv.Atoi(httpRange[0])- if err != nil {- return fmt.Errorf("unable to convert lower bound to int %q", successRange)+ var raw string+ for _, values := range match {+ // values[0] contains the entire regex match.+ raw += values[0]}-- upperBound, err := strconv.Atoi(httpRange[1])- if err != nil {- return fmt.Errorf("unable to convert upper bound to int %q", successRange)+ result := detectors.Result{+ DetectorType: detectorspb.DetectorType_CustomRegex,+ Raw: []byte(raw),}- if lowerBound > upperBound {- return fmt.Errorf("lower bound greater than upper bound on range %q", successRange)+ if isKnownFalsePositive(match) {+ continue}-- if lowerBound < httpLowerBound || upperBound > httpUpperBound {- return fmt.Errorf("invalid http status code range %q", successRange)+ if !verify {+ results = append(results, result)+ continue}- }- return nil-}--func ValidateRegexVars(regex map[string]string, body ...string) error {- for _, b := range body {- matches := NewRegexVarString(b).variables-- for match := range matches {- if _, ok := regex[match]; !ok {- return fmt.Errorf("body %q contains an unknown variable", b)+ // Verify via webhook.+ jsonBody, err := json.Marshal(map[string]map[string][]string{+ c.GetName(): match,+ })+ if err != nil {+ continue+ }+ // Try each config until we successfully verify.+ for _, verifyConfig := range c.GetVerify() {+ if common.IsDone(ctx) {+ // TODO: Log we're possibly leaving out results.+ return results, nil+ }+ req, err := http.NewRequestWithContext(ctx, "POST", verifyConfig.GetEndpoint(), bytes.NewReader(jsonBody))+ if err != nil {+ continue+ }+ for _, header := range verifyConfig.GetHeaders() {+ key, value, found := strings.Cut(header, ":")+ if !found {+ // Should be unreachable due to validation.+ continue+ }+ req.Header.Add(key, strings.TrimLeft(value, "\t\n\v\f\r "))+ }+ res, err := httpClient.Do(req)+ if err != nil {+ continue+ }+ // TODO: Read response body.+ res.Body.Close()+ if res.StatusCode == http.StatusOK {+ result.Verified = true+ break}}+ results = append(results, result)}- return nil+ return results, nil}-func NewCustomRegex(pb *custom_detectorspb.CustomRegex) (customRegex, error) {- // TODO: Return all validation errors.- if err := ValidateKeywords(pb.Keywords); err != nil {- return nil, err- }+func (c *customRegexWebhook) Keywords() []string {+ return c.GetKeywords()+}- if err := ValidateRegex(pb.Regex); err != nil {- return nil, err+// productIndices produces a permutation of indices for each length. Example:+// productIndices(3, 2) -> [[0 0] [1 0] [2 0] [0 1] [1 1] [2 1]]. It returns+// a slice of length no larger than maxTotalMatches.+func productIndices(lengths ...int) [][]int {+ count := 1+ for _, l := range lengths {+ count *= l+ }+ if count == 0 {+ return nil+ }+ if count > maxTotalMatches {+ count = maxTotalMatches}- for _, verify := range pb.Verify {-- if err := ValidateVerifyEndpoint(verify.Endpoint, verify.Unsafe); err != nil {- return nil, err+ results := make([][]int, count)+ for i := 0; i < count; i++ {+ j := 1+ result := make([]int, 0, len(lengths))+ for _, l := range lengths {+ result = append(result, (i/j)%l)+ j *= l}+ results[i] = result+ }+ return results+}- if err := ValidateVerifyHeaders(verify.Headers); err != nil {- return nil, err- }+// permutateMatches converts the list of all regex matches into all possible+// permutations selecting one from each named entry in the map. For example:+// {"foo": [matchA, matchB], "bar": [matchC]} becomes+// [{"foo": matchA, "bar": matchC}, {"foo": matchB, "bar": matchC}]+func permutateMatches(regexMatches map[string][][]string) []map[string][]string {+ // Get a consistent order for names and their matching lengths.+ // The lengths are used in calculating the permutation so order matters.+ names := make([]string, 0, len(regexMatches))+ lengths := make([]int, 0, len(regexMatches))+ for key, value := range regexMatches {+ names = append(names, key)+ lengths = append(lengths, len(value))+ }- if err := ValidateVerifyRanges(verify.SuccessRanges); err != nil {- return nil, err+ // Permutate all the indices for each match. For example, if "foo" has+ // [matchA, matchB] and "bar" has [matchC], we will get indices [0 0] [1 0].+ permutationIndices := productIndices(lengths...)++ // Build {"foo": matchA, "bar": matchC} and {"foo": matchB, "bar": matchC}+ // from the indices.+ var matches []map[string][]string+ for _, permutation := range permutationIndices {+ candidate := make(map[string][]string, len(permutationIndices))+ for i, name := range names {+ candidate[name] = regexMatches[name][permutation[i]]}+ matches = append(matches, candidate)+ }- if err := ValidateRegexVars(pb.Regex, append(verify.Headers, verify.Endpoint)...); err != nil {- return nil, err- }+ return matches+}+// This function will check false positives for common test words, but also it+// will make sure the key appears 'random' enough to be a real key.+func isKnownFalsePositive(match map[string][]string) bool {+ for _, values := range match {+ for _, value := range values {+ if detectors.IsKnownFalsePositive(value, detectors.DefaultFalsePositives, true) {+ return true+ }+ }}-- return pb, nil+ return false}diff --git a/pkg/custom_detectors/custom_detectors_test.go b/pkg/custom_detectors/custom_detectors_test.goindex 834f3d3d8..7b4fa85f8 100644--- a/pkg/custom_detectors/custom_detectors_test.go+++ b/pkg/custom_detectors/custom_detectors_test.go@@ -102,226 +102,82 @@ func TestCustomDetectorsParsing(t *testing.T) {assert.Equal(t, []string{"Authorization: Bearer token"}, got.Verify[0].Headers)}-func TestCustomDetectorsKeywordValidation(t *testing.T) {+func TestProductIndices(t *testing.T) {tests := []struct {- name string- input []string- wantErr bool+ name string+ input []int+ want [][]int}{{- name: "Test empty list of keywords",- input: []string{},- wantErr: true,+ name: "zero",+ input: []int{3, 0},+ want: nil,},{- name: "Test empty keyword",- input: []string{""},- wantErr: true,+ name: "one input",+ input: []int{3},+ want: [][]int{{0}, {1}, {2}},},{- name: "Test valid keywords",- input: []string{"hello", "world"},- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := ValidateKeywords(tt.input)-- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {- t.Errorf("ValidateKeywords() error = %v, wantErr %v", got, tt.wantErr)- }- })- }-}--func TestCustomDetectorsRegexValidation(t *testing.T) {- tests := []struct {- name string- input map[string]string- wantErr bool- }{- {- name: "Test list of keywords",- input: map[string]string{- "id_pat_example": "([a-zA-Z0-9]{32})",+ name: "two inputs",+ input: []int{3, 2},+ want: [][]int{+ {0, 0}, {1, 0}, {2, 0},+ {0, 1}, {1, 1}, {2, 1},},- wantErr: false,},{- name: "Test empty list of keywords",- input: map[string]string{},- wantErr: true,+ name: "three inputs",+ input: []int{3, 2, 3},+ want: [][]int{+ {0, 0, 0}, {1, 0, 0}, {2, 0, 0},+ {0, 1, 0}, {1, 1, 0}, {2, 1, 0},+ {0, 0, 1}, {1, 0, 1}, {2, 0, 1},+ {0, 1, 1}, {1, 1, 1}, {2, 1, 1},+ {0, 0, 2}, {1, 0, 2}, {2, 0, 2},+ {0, 1, 2}, {1, 1, 2}, {2, 1, 2},+ },},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got := ValidateRegex(tt.input)-- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {- t.Errorf("ValidateRegex() error = %v, wantErr %v", got, tt.wantErr)- }+ got := productIndices(tt.input...)+ assert.Equal(t, tt.want, got)})}}-func TestCustomDetectorsVerifyEndpointValidation(t *testing.T) {- tests := []struct {- name string- endpoint string- unsafe bool- wantErr bool- }{- {- name: "Test http endpoint with unsafe flag",- endpoint: "http://localhost:8000/{id_pat_example}",- unsafe: true,- wantErr: false,- },- {- name: "Test http endpoint without unsafe flag",- endpoint: "http://localhost:8000/{id_pat_example}",- unsafe: false,- wantErr: true,- },- {- name: "Test https endpoint with unsafe flag",- endpoint: "https://localhost:8000/{id_pat_example}",- unsafe: true,- wantErr: false,- },- {- name: "Test https endpoint without unsafe flag",- endpoint: "https://localhost:8000/{id_pat_example}",- unsafe: false,- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := ValidateVerifyEndpoint(tt.endpoint, tt.unsafe)-- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {- t.Errorf("ValidateVerifyEndpoint() error = %v, wantErr %v", got, tt.wantErr)- }- })- }+func TestProductIndicesMax(t *testing.T) {+ got := productIndices(2, 3, 4, 5, 6)+ assert.GreaterOrEqual(t, 2*3*4*5*6, maxTotalMatches)+ assert.Equal(t, maxTotalMatches, len(got))}-func TestCustomDetectorsVerifyHeadersValidation(t *testing.T) {+func TestPermutateMatches(t *testing.T) {tests := []struct {- name string- headers []string- wantErr bool+ name string+ input map[string][][]string+ want []map[string][]string}{{- name: "Test single header",- headers: []string{"Authorization: Bearer {secret_pat_example.0}"},- wantErr: false,- },- {- name: "Test invalid header",- headers: []string{"Hello world"},- wantErr: true,- },- {- name: "Test ugly header",- headers: []string{"Hello:::::::world::hi:"},- wantErr: false,- },- {- name: "Test empty header",- headers: []string{},- wantErr: false,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := ValidateVerifyHeaders(tt.headers)-- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {- t.Errorf("ValidateVerifyHeaders() error = %v, wantErr %v", got, tt.wantErr)- }- })- }-}--func TestCustomDetectorsVerifyRangeValidation(t *testing.T) {- tests := []struct {- name string- ranges []string- wantErr bool- }{- {- name: "Test multiple mixed ranges",- ranges: []string{"200", "300-350"},- wantErr: false,- },- {- name: "Test invalid non-number range",- ranges: []string{"hi"},- wantErr: true,- },- {- name: "Test invalid lower to upper range",- ranges: []string{"200-100"},- wantErr: true,- },- {- name: "Test invalid http range",- ranges: []string{"400-1000"},- wantErr: true,- },- {- name: "Test multiple ranges with invalid inputs",- ranges: []string{"322", "hello-world", "100-200"},- wantErr: true,+ name: "two matches",+ input: map[string][][]string{"foo": {{"matchA"}, {"matchB"}}, "bar": {{"matchC"}}},+ want: []map[string][]string{+ {"foo": {"matchA"}, "bar": {"matchC"}},+ {"foo": {"matchB"}, "bar": {"matchC"}},+ },},}for _, tt := range tests {t.Run(tt.name, func(t *testing.T) {- got := ValidateVerifyRanges(tt.ranges)-- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {- t.Errorf("ValidateVerifyRanges() error = %v, wantErr %v", got, tt.wantErr)- }+ got := permutateMatches(tt.input)+ assert.Equal(t, tt.want, got)})}}-func TestCustomDetectorsVerifyRegexVarsValidation(t *testing.T) {- tests := []struct {- name string- regex map[string]string- body string- wantErr bool- }{- {- name: "Regex defined but not used in body",- regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},- body: "hello world",- wantErr: false,- },- {- name: "Regex defined and is used in body",- regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},- body: "hello world {id}",- wantErr: false,- },- {- name: "Regex var in body but not defined",- regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},- body: "hello world {hello}",- wantErr: true,- },- }- for _, tt := range tests {- t.Run(tt.name, func(t *testing.T) {- got := ValidateRegexVars(tt.regex, tt.body)-- if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {- t.Errorf("ValidateRegexVars() error = %v, wantErr %v", got, tt.wantErr)- }- })+func BenchmarkProductIndices(b *testing.B) {+ for i := 0; i < b.N; i++ {+ _ = productIndices(3, 2, 6)}}diff --git a/pkg/custom_detectors/validation.go b/pkg/custom_detectors/validation.gonew file mode 100644index 000000000..62bc14910--- /dev/null+++ b/pkg/custom_detectors/validation.go@@ -0,0 +1,103 @@+package custom_detectors++import (+ "fmt"+ "strconv"+ "strings"+)++func ValidateKeywords(keywords []string) error {+ if len(keywords) == 0 {+ return fmt.Errorf("no keywords")+ }++ for _, keyword := range keywords {+ if len(keyword) == 0 {+ return fmt.Errorf("empty keyword")+ }+ }+ return nil+}++func ValidateRegex(regex map[string]string) error {+ if len(regex) == 0 {+ return fmt.Errorf("no regex")+ }+ return nil+}++func ValidateVerifyEndpoint(endpoint string, unsafe bool) error {+ if len(endpoint) == 0 {+ return fmt.Errorf("no endpoint")+ }++ if strings.HasPrefix(endpoint, "http://") && !unsafe {+ return fmt.Errorf("http endpoint must have unsafe=true")+ }+ return nil+}++func ValidateVerifyHeaders(headers []string) error {+ for _, header := range headers {+ if !strings.Contains(header, ":") {+ return fmt.Errorf("header %q must contain a colon", header)+ }+ }+ return nil+}++func ValidateVerifyRanges(ranges []string) error {+ const httpLowerRange = 100+ const httpUpperRange = 599++ for _, successRange := range ranges {+ if !strings.Contains(successRange, "-") {+ httpCode, err := strconv.Atoi(successRange)+ if err != nil {+ return fmt.Errorf("unable to convert http code to int %q", successRange)+ }++ if httpCode < httpLowerRange || httpCode > httpUpperRange {+ return fmt.Errorf("invalid http status code %q", successRange)+ }++ continue+ }++ httpRange := strings.Split(successRange, "-")+ if len(httpRange) != 2 {+ return fmt.Errorf("invalid range format %q", successRange)+ }++ lowerBound, err := strconv.Atoi(httpRange[0])+ if err != nil {+ return fmt.Errorf("unable to convert lower bound to int %q", successRange)+ }++ upperBound, err := strconv.Atoi(httpRange[1])+ if err != nil {+ return fmt.Errorf("unable to convert upper bound to int %q", successRange)+ }++ if lowerBound > upperBound {+ return fmt.Errorf("lower bound greater than upper bound on range %q", successRange)+ }++ if lowerBound < httpLowerRange || upperBound > httpUpperRange {+ return fmt.Errorf("invalid http status code range %q", successRange)+ }+ }+ return nil+}++func ValidateRegexVars(regex map[string]string, body ...string) error {+ for _, b := range body {+ matches := NewRegexVarString(b).variables+ for match := range matches {+ if _, ok := regex[match]; !ok {+ return fmt.Errorf("body %q contains an unknown variable", b)+ }+ }+ }+ return nil+}diff --git a/pkg/custom_detectors/validation_test.go b/pkg/custom_detectors/validation_test.gonew file mode 100644index 000000000..1aa36f4e0--- /dev/null+++ b/pkg/custom_detectors/validation_test.go@@ -0,0 +1,227 @@+package custom_detectors++import "testing"++func TestCustomDetectorsKeywordValidation(t *testing.T) {+ tests := []struct {+ name string+ input []string+ wantErr bool+ }{+ {+ name: "Test empty list of keywords",+ input: []string{},+ wantErr: true,+ },+ {+ name: "Test empty keyword",+ input: []string{""},+ wantErr: true,+ },+ {+ name: "Test valid keywords",+ input: []string{"hello", "world"},+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := ValidateKeywords(tt.input)++ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {+ t.Errorf("ValidateKeywords() error = %v, wantErr %v", got, tt.wantErr)+ }+ })+ }+}++func TestCustomDetectorsRegexValidation(t *testing.T) {+ tests := []struct {+ name string+ input map[string]string+ wantErr bool+ }{+ {+ name: "Test list of keywords",+ input: map[string]string{+ "id_pat_example": "([a-zA-Z0-9]{32})",+ },+ wantErr: false,+ },+ {+ name: "Test empty list of keywords",+ input: map[string]string{},+ wantErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := ValidateRegex(tt.input)++ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {+ t.Errorf("ValidateRegex() error = %v, wantErr %v", got, tt.wantErr)+ }+ })+ }+}++func TestCustomDetectorsVerifyEndpointValidation(t *testing.T) {+ tests := []struct {+ name string+ endpoint string+ unsafe bool+ wantErr bool+ }{+ {+ name: "Test http endpoint with unsafe flag",+ endpoint: "http://localhost:8000/{id_pat_example}",+ unsafe: true,+ wantErr: false,+ },+ {+ name: "Test http endpoint without unsafe flag",+ endpoint: "http://localhost:8000/{id_pat_example}",+ unsafe: false,+ wantErr: true,+ },+ {+ name: "Test https endpoint with unsafe flag",+ endpoint: "https://localhost:8000/{id_pat_example}",+ unsafe: true,+ wantErr: false,+ },+ {+ name: "Test https endpoint without unsafe flag",+ endpoint: "https://localhost:8000/{id_pat_example}",+ unsafe: false,+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := ValidateVerifyEndpoint(tt.endpoint, tt.unsafe)++ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {+ t.Errorf("ValidateVerifyEndpoint() error = %v, wantErr %v", got, tt.wantErr)+ }+ })+ }+}++func TestCustomDetectorsVerifyHeadersValidation(t *testing.T) {+ tests := []struct {+ name string+ headers []string+ wantErr bool+ }{+ {+ name: "Test single header",+ headers: []string{"Authorization: Bearer {secret_pat_example.0}"},+ wantErr: false,+ },+ {+ name: "Test invalid header",+ headers: []string{"Hello world"},+ wantErr: true,+ },+ {+ name: "Test ugly header",+ headers: []string{"Hello:::::::world::hi:"},+ wantErr: false,+ },+ {+ name: "Test empty header",+ headers: []string{},+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := ValidateVerifyHeaders(tt.headers)++ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {+ t.Errorf("ValidateVerifyHeaders() error = %v, wantErr %v", got, tt.wantErr)+ }+ })+ }+}++func TestCustomDetectorsVerifyRangeValidation(t *testing.T) {+ tests := []struct {+ name string+ ranges []string+ wantErr bool+ }{+ {+ name: "Test multiple mixed ranges",+ ranges: []string{"200", "300-350"},+ wantErr: false,+ },+ {+ name: "Test invalid non-number range",+ ranges: []string{"hi"},+ wantErr: true,+ },+ {+ name: "Test invalid lower to upper range",+ ranges: []string{"200-100"},+ wantErr: true,+ },+ {+ name: "Test invalid http range",+ ranges: []string{"400-1000"},+ wantErr: true,+ },+ {+ name: "Test multiple ranges with invalid inputs",+ ranges: []string{"322", "hello-world", "100-200"},+ wantErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := ValidateVerifyRanges(tt.ranges)++ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {+ t.Errorf("ValidateVerifyRanges() error = %v, wantErr %v", got, tt.wantErr)+ }+ })+ }+}++func TestCustomDetectorsVerifyRegexVarsValidation(t *testing.T) {+ tests := []struct {+ name string+ regex map[string]string+ body string+ wantErr bool+ }{+ {+ name: "Regex defined but not used in body",+ regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},+ body: "hello world",+ wantErr: false,+ },+ {+ name: "Regex defined and is used in body",+ regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},+ body: "hello world {id}",+ wantErr: false,+ },+ {+ name: "Regex var in body but not defined",+ regex: map[string]string{"id": "[0-9]{1,10}", "id_pat_example": "([a-zA-Z0-9]{32})"},+ body: "hello world {hello}",+ wantErr: true,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := ValidateRegexVars(tt.regex, tt.body)++ if (got != nil && !tt.wantErr) || (got == nil && tt.wantErr) {+ t.Errorf("ValidateRegexVars() error = %v, wantErr %v", got, tt.wantErr)+ }+ })+ }+}diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.goindex d4012ae06..f81028492 100644--- a/pkg/pb/detectorspb/detectors.pb.go+++ b/pkg/pb/detectorspb/detectors.pb.go@@ -972,6 +972,7 @@ const (DetectorType_LDAP DetectorType = 901DetectorType_Shopify DetectorType = 902DetectorType_RabbitMQ DetectorType = 903+ DetectorType_CustomRegex DetectorType = 904)// Enum value maps for DetectorType.@@ -1877,6 +1878,7 @@ var (901: "LDAP",902: "Shopify",903: "RabbitMQ",+ 904: "CustomRegex",}DetectorType_value = map[string]int32{"Alibaba": 0,@@ -2779,6 +2781,7 @@ var ("LDAP": 901,"Shopify": 902,"RabbitMQ": 903,+ "CustomRegex": 904,})@@ -3143,7 +3146,7 @@ var file_detectors_proto_rawDesc = []byte{0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,- 0x02, 0x2a, 0xff, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,+ 0x02, 0x2a, 0x91, 0x71, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12,0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53,0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a,@@ -4047,11 +4050,12 @@ var file_detectors_proto_rawDesc = []byte{0x0a, 0x0a, 0x05, 0x52, 0x65, 0x64, 0x69, 0x73, 0x10, 0x84, 0x07, 0x12, 0x09, 0x0a, 0x04, 0x4c,0x44, 0x41, 0x50, 0x10, 0x85, 0x07, 0x12, 0x0c, 0x0a, 0x07, 0x53, 0x68, 0x6f, 0x70, 0x69, 0x66,0x79, 0x10, 0x86, 0x07, 0x12, 0x0d, 0x0a, 0x08, 0x52, 0x61, 0x62, 0x62, 0x69, 0x74, 0x4d, 0x51,- 0x10, 0x87, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f,- 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74,- 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f,- 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73,- 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,+ 0x10, 0x87, 0x07, 0x12, 0x10, 0x0a, 0x0b, 0x43, 0x75, 0x73, 0x74, 0x6f, 0x6d, 0x52, 0x65, 0x67,+ 0x65, 0x78, 0x10, 0x88, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,+ 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72,+ 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76,+ 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f,+ 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,}var (diff --git a/proto/detectors.proto b/proto/detectors.protoindex 0b86a59f5..482084c95 100644--- a/proto/detectors.proto+++ b/proto/detectors.proto@@ -911,6 +911,7 @@ enum DetectorType {LDAP = 901;Shopify = 902;RabbitMQ = 903;+ CustomRegex = 904;}message Result {