Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
[INS-228] Add ignorePattern configuration support to Postgres and Sqlserver detectors (#4612)
Mustansir7f4e37db2d92
diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.goindex f22291a6e..3a7449e05 100644--- a/pkg/detectors/postgres/postgres.go+++ b/pkg/detectors/postgres/postgres.go@@ -55,6 +55,33 @@ var (type Scanner struct {detectors.DefaultMultiPartCredentialProviderdetectLoopback bool // Automated tests run against localhost, but we want to ignore those results in the wild+ ignorePatterns []*regexp.Regexp+}++func New(opts ...func(*Scanner)) *Scanner {+ scanner := &Scanner{+ ignorePatterns: []*regexp.Regexp{},+ }+ for _, opt := range opts {+ opt(scanner)+ }++ return scanner+}++func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {+ return func(s *Scanner) {+ var ignorePatterns []*regexp.Regexp+ for _, ignoreString := range ignoreStrings {+ ignorePattern, err := regexp.Compile(ignoreString)+ if err != nil {+ panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err))+ }+ ignorePatterns = append(ignorePatterns, ignorePattern)+ }++ s.ignorePatterns = ignorePatterns+ }}var _ detectors.Detector = (*Scanner)(nil)@@ -66,7 +93,7 @@ func (s Scanner) Keywords() []string {func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {var results []detectors.Result- candidateParamSets := findUriMatches(data)+ candidateParamSets := findUriMatches(data, s.ignorePatterns)for _, params := range candidateParamSets {if common.IsDone(ctx) {@@ -161,9 +188,12 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {return false, ""}-func findUriMatches(data []byte) []map[string]string {+func findUriMatches(data []byte, ignorePatterns []*regexp.Regexp) []map[string]string {var matches []map[string]stringfor _, uri := range uriPattern.FindAll(data, -1) {+ if shouldIgnore(uri, ignorePatterns) {+ continue+ }// Capture the database type (e.g., "postgres" or "postgresql")dbTypeMatch := uriPattern.FindSubmatch(uri)if len(dbTypeMatch) < 2 {@@ -188,6 +218,15 @@ func findUriMatches(data []byte) []map[string]string {return matches}+func shouldIgnore(uri []byte, ignorePatterns []*regexp.Regexp) bool {+ for _, ignore := range ignorePatterns {+ if ignore.Match(uri) {+ return true+ }+ }+ return false+}+// getDeadlineInSeconds gets the deadline from the context in seconds. If there// is no deadline, false is returned. If the deadline is already exceeded, a// negative or 0 value will be returned.diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.goindex 0570cca9a..85f5b82eb 100644--- a/pkg/detectors/postgres/postgres_test.go+++ b/pkg/detectors/postgres/postgres_test.go@@ -81,3 +81,18 @@ func TestPostgres_Pattern(t *testing.T) {})}}++func TestPostgres_FromDataWithIgnorePattern(t *testing.T) {+ s := New(+ WithIgnorePattern([]string{+ `1\.2\.3\.4`,+ }))+ got, err := s.FromData(context.Background(), false, []byte(validUriPattern))+ if err != nil {+ t.Errorf("FromData() error = %v", err)+ return+ }+ if len(got) != 0 {+ t.Errorf("expected no results, but got %d", len(got))+ }+}diff --git a/pkg/detectors/sqlserver/sqlserver.go b/pkg/detectors/sqlserver/sqlserver.goindex 283710752..78d6bcd6a 100644--- a/pkg/detectors/sqlserver/sqlserver.go+++ b/pkg/detectors/sqlserver/sqlserver.go@@ -18,7 +18,35 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors")-type Scanner struct{}+type Scanner struct {+ ignorePatterns []*regexp.Regexp+}++func New(opts ...func(*Scanner)) *Scanner {+ scanner := &Scanner{+ ignorePatterns: []*regexp.Regexp{},+ }+ for _, opt := range opts {+ opt(scanner)+ }++ return scanner+}++func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {+ return func(s *Scanner) {+ var ignorePatterns []*regexp.Regexp+ for _, ignoreString := range ignoreStrings {+ ignorePattern, err := regexp.Compile(ignoreString)+ if err != nil {+ panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err))+ }+ ignorePatterns = append(ignorePatterns, ignorePattern)+ }++ s.ignorePatterns = ignorePatterns+ }+}// Ensure the Scanner satisfies the interface at compile time.var _ detectors.Detector = (*Scanner)(nil)@@ -38,6 +66,9 @@ func (s Scanner) Keywords() []string {func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {matches := pattern.FindAllStringSubmatch(string(data), -1)for _, match := range matches {+ if shouldIgnore(match[1], s.ignorePatterns) {+ continue+ }paramsUnsafe, err := msdsn.Parse(match[1])if err != nil {continue@@ -80,6 +111,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}+func shouldIgnore(uri string, ignorePatterns []*regexp.Regexp) bool {+ for _, ignore := range ignorePatterns {+ if ignore.MatchString(uri) {+ return true+ }+ }+ return false+}+var ping = func(ctx context.Context, config msdsn.Config) (bool, error) {// TCP connectivity check to prevent indefinite hangsaddress := net.JoinHostPort(config.Host, strconv.Itoa(int(config.Port)))diff --git a/pkg/detectors/sqlserver/sqlserver_test.go b/pkg/detectors/sqlserver/sqlserver_test.goindex fc5748b71..8c376eb06 100644--- a/pkg/detectors/sqlserver/sqlserver_test.go+++ b/pkg/detectors/sqlserver/sqlserver_test.go@@ -1,7 +1,13 @@package sqlserverimport (+ "context"+ "fmt""testing"++ "github.com/google/go-cmp/cmp"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick")func TestSQLServer_Pattern(t *testing.T) {@@ -18,3 +24,87 @@ func TestSQLServer_Pattern(t *testing.T) {t.Errorf("SQLServer.pattern: did not find connection string in xml format")}}++const (+ validConnStr = `builder.Services.AddDbContext<Database>(optionsBuilder => optionsBuilder.UseSqlServer("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"));`+ validParsedStr = `sqlserver://sa:P%40ssw0rd%21@localhost?database=master&dial+timeout=15&disableretry=false`+ invalidConnStr = `some random text without connection string`+)++func TestSqlServer_Pattern(t *testing.T) {+ d := Scanner{}+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})+ tests := []struct {+ name string+ input string+ want []string+ }{+ {+ name: "valid pattern - with keyword sql",+ input: fmt.Sprintf(`sql - %s`, validConnStr),+ want: []string{validParsedStr},+ },+ {+ name: "invalid pattern",+ input: fmt.Sprintf("sql=%s", invalidConnStr),+ want: []string{},+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))+ if len(matchedDetectors) == 0 {+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)+ return+ }++ results, err := d.FromData(context.Background(), false, []byte(test.input))+ if err != nil {+ t.Errorf("error = %v", err)+ return+ }++ if len(results) != len(test.want) {+ if len(results) == 0 {+ t.Errorf("did not receive result")+ } else {+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))+ }+ return+ }++ actual := make(map[string]struct{}, len(results))+ for _, r := range results {+ if len(r.RawV2) > 0 {+ actual[string(r.RawV2)] = struct{}{}+ } else {+ actual[string(r.Raw)] = struct{}{}+ }+ }+ expected := make(map[string]struct{}, len(test.want))+ for _, v := range test.want {+ expected[v] = struct{}{}+ }++ if diff := cmp.Diff(expected, actual); diff != "" {+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)+ }+ })+ }+}++func TestSqlServer_FromDataWithIgnorePattern(t *testing.T) {+ s := New(+ WithIgnorePattern([]string{+ `^Server=localhost`,+ }))+ got, err := s.FromData(context.Background(), false, []byte("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true"))+ if err != nil {+ t.Errorf("FromData() error = %v", err)+ return+ }+ if len(got) != 0 {+ t.Errorf("expected no results, but got %d", len(got))+ }+}