Skip to content

Find, verify, and analyze leaked credentials

[INS-228] Add ignorePattern configuration support to Postgres and Sqlserver detectors (#4612)

Mustansir7f4e37db2d92
diff --git a/pkg/detectors/postgres/postgres.go b/pkg/detectors/postgres/postgres.go
index f22291a6e..3a7449e05 100644
--- a/pkg/detectors/postgres/postgres.go
+++ b/pkg/detectors/postgres/postgres.go
@@ -55,6 +55,33 @@ var (
type Scanner struct {
detectors.DefaultMultiPartCredentialProvider
detectLoopback bool // Automated tests run against localhost, but we want to ignore those results in the wild
+ ignorePatterns []*regexp.Regexp
+}
+
+func New(opts ...func(*Scanner)) *Scanner {
+ scanner := &Scanner{
+ ignorePatterns: []*regexp.Regexp{},
+ }
+ for _, opt := range opts {
+ opt(scanner)
+ }
+
+ return scanner
+}
+
+func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {
+ return func(s *Scanner) {
+ var ignorePatterns []*regexp.Regexp
+ for _, ignoreString := range ignoreStrings {
+ ignorePattern, err := regexp.Compile(ignoreString)
+ if err != nil {
+ panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err))
+ }
+ ignorePatterns = append(ignorePatterns, ignorePattern)
+ }
+
+ s.ignorePatterns = ignorePatterns
+ }
}
var _ detectors.Detector = (*Scanner)(nil)
@@ -66,7 +93,7 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) ([]detectors.Result, error) {
var results []detectors.Result
- candidateParamSets := findUriMatches(data)
+ candidateParamSets := findUriMatches(data, s.ignorePatterns)
for _, params := range candidateParamSets {
if common.IsDone(ctx) {
@@ -161,9 +188,12 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {
return false, ""
}
-func findUriMatches(data []byte) []map[string]string {
+func findUriMatches(data []byte, ignorePatterns []*regexp.Regexp) []map[string]string {
var matches []map[string]string
for _, uri := range uriPattern.FindAll(data, -1) {
+ if shouldIgnore(uri, ignorePatterns) {
+ continue
+ }
// Capture the database type (e.g., "postgres" or "postgresql")
dbTypeMatch := uriPattern.FindSubmatch(uri)
if len(dbTypeMatch) < 2 {
@@ -188,6 +218,15 @@ func findUriMatches(data []byte) []map[string]string {
return matches
}
+func shouldIgnore(uri []byte, ignorePatterns []*regexp.Regexp) bool {
+ for _, ignore := range ignorePatterns {
+ if ignore.Match(uri) {
+ return true
+ }
+ }
+ return false
+}
+
// getDeadlineInSeconds gets the deadline from the context in seconds. If there
// is no deadline, false is returned. If the deadline is already exceeded, a
// negative or 0 value will be returned.
diff --git a/pkg/detectors/postgres/postgres_test.go b/pkg/detectors/postgres/postgres_test.go
index 0570cca9a..85f5b82eb 100644
--- a/pkg/detectors/postgres/postgres_test.go
+++ b/pkg/detectors/postgres/postgres_test.go
@@ -81,3 +81,18 @@ func TestPostgres_Pattern(t *testing.T) {
})
}
}
+
+func TestPostgres_FromDataWithIgnorePattern(t *testing.T) {
+ s := New(
+ WithIgnorePattern([]string{
+ `1\.2\.3\.4`,
+ }))
+ got, err := s.FromData(context.Background(), false, []byte(validUriPattern))
+ if err != nil {
+ t.Errorf("FromData() error = %v", err)
+ return
+ }
+ if len(got) != 0 {
+ t.Errorf("expected no results, but got %d", len(got))
+ }
+}
diff --git a/pkg/detectors/sqlserver/sqlserver.go b/pkg/detectors/sqlserver/sqlserver.go
index 283710752..78d6bcd6a 100644
--- a/pkg/detectors/sqlserver/sqlserver.go
+++ b/pkg/detectors/sqlserver/sqlserver.go
@@ -18,7 +18,35 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
)
-type Scanner struct{}
+type Scanner struct {
+ ignorePatterns []*regexp.Regexp
+}
+
+func New(opts ...func(*Scanner)) *Scanner {
+ scanner := &Scanner{
+ ignorePatterns: []*regexp.Regexp{},
+ }
+ for _, opt := range opts {
+ opt(scanner)
+ }
+
+ return scanner
+}
+
+func WithIgnorePattern(ignoreStrings []string) func(*Scanner) {
+ return func(s *Scanner) {
+ var ignorePatterns []*regexp.Regexp
+ for _, ignoreString := range ignoreStrings {
+ ignorePattern, err := regexp.Compile(ignoreString)
+ if err != nil {
+ panic(fmt.Sprintf("%s is not a valid regex, error received: %v", ignoreString, err))
+ }
+ ignorePatterns = append(ignorePatterns, ignorePattern)
+ }
+
+ s.ignorePatterns = ignorePatterns
+ }
+}
// Ensure the Scanner satisfies the interface at compile time.
var _ detectors.Detector = (*Scanner)(nil)
@@ -38,6 +66,9 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
matches := pattern.FindAllStringSubmatch(string(data), -1)
for _, match := range matches {
+ if shouldIgnore(match[1], s.ignorePatterns) {
+ continue
+ }
paramsUnsafe, err := msdsn.Parse(match[1])
if err != nil {
continue
@@ -80,6 +111,15 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return results, nil
}
+func shouldIgnore(uri string, ignorePatterns []*regexp.Regexp) bool {
+ for _, ignore := range ignorePatterns {
+ if ignore.MatchString(uri) {
+ return true
+ }
+ }
+ return false
+}
+
var ping = func(ctx context.Context, config msdsn.Config) (bool, error) {
// TCP connectivity check to prevent indefinite hangs
address := net.JoinHostPort(config.Host, strconv.Itoa(int(config.Port)))
diff --git a/pkg/detectors/sqlserver/sqlserver_test.go b/pkg/detectors/sqlserver/sqlserver_test.go
index fc5748b71..8c376eb06 100644
--- a/pkg/detectors/sqlserver/sqlserver_test.go
+++ b/pkg/detectors/sqlserver/sqlserver_test.go
@@ -1,7 +1,13 @@
package sqlserver
import (
+ "context"
+ "fmt"
"testing"
+
+ "github.com/google/go-cmp/cmp"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/engine/ahocorasick"
)
func TestSQLServer_Pattern(t *testing.T) {
@@ -18,3 +24,87 @@ func TestSQLServer_Pattern(t *testing.T) {
t.Errorf("SQLServer.pattern: did not find connection string in xml format")
}
}
+
+const (
+ validConnStr = `builder.Services.AddDbContext<Database>(optionsBuilder => optionsBuilder.UseSqlServer("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true;"));`
+ validParsedStr = `sqlserver://sa:P%40ssw0rd%21@localhost?database=master&dial+timeout=15&disableretry=false`
+ invalidConnStr = `some random text without connection string`
+)
+
+func TestSqlServer_Pattern(t *testing.T) {
+ d := Scanner{}
+ ahoCorasickCore := ahocorasick.NewAhoCorasickCore([]detectors.Detector{d})
+ tests := []struct {
+ name string
+ input string
+ want []string
+ }{
+ {
+ name: "valid pattern - with keyword sql",
+ input: fmt.Sprintf(`sql - %s`, validConnStr),
+ want: []string{validParsedStr},
+ },
+ {
+ name: "invalid pattern",
+ input: fmt.Sprintf("sql=%s", invalidConnStr),
+ want: []string{},
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ matchedDetectors := ahoCorasickCore.FindDetectorMatches([]byte(test.input))
+ if len(matchedDetectors) == 0 {
+ t.Errorf("keywords '%v' not matched by: %s", d.Keywords(), test.input)
+ return
+ }
+
+ results, err := d.FromData(context.Background(), false, []byte(test.input))
+ if err != nil {
+ t.Errorf("error = %v", err)
+ return
+ }
+
+ if len(results) != len(test.want) {
+ if len(results) == 0 {
+ t.Errorf("did not receive result")
+ } else {
+ t.Errorf("expected %d results, only received %d", len(test.want), len(results))
+ }
+ return
+ }
+
+ actual := make(map[string]struct{}, len(results))
+ for _, r := range results {
+ if len(r.RawV2) > 0 {
+ actual[string(r.RawV2)] = struct{}{}
+ } else {
+ actual[string(r.Raw)] = struct{}{}
+ }
+ }
+ expected := make(map[string]struct{}, len(test.want))
+ for _, v := range test.want {
+ expected[v] = struct{}{}
+ }
+
+ if diff := cmp.Diff(expected, actual); diff != "" {
+ t.Errorf("%s diff: (-want +got)\n%s", test.name, diff)
+ }
+ })
+ }
+}
+
+func TestSqlServer_FromDataWithIgnorePattern(t *testing.T) {
+ s := New(
+ WithIgnorePattern([]string{
+ `^Server=localhost`,
+ }))
+ got, err := s.FromData(context.Background(), false, []byte("Server=localhost;Initial Catalog=master;User ID=sa;Password=P@ssw0rd!;Persist Security Info=true;MultipleActiveResultSets=true"))
+ if err != nil {
+ t.Errorf("FromData() error = %v", err)
+ return
+ }
+ if len(got) != 0 {
+ t.Errorf("expected no results, but got %d", len(got))
+ }
+}