Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Improve MongoDB connection string matching (#1550)
Richard Gomez75557f61edd5
diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.goindex 8593fae01..22bced1fa 100644--- a/pkg/detectors/mongodb/mongodb.go+++ b/pkg/detectors/mongodb/mongodb.go@@ -2,6 +2,7 @@ package mongodbimport ("context"+ "errors""net/url""strings""time"@@ -12,7 +13,6 @@ import ("go.mongodb.org/mongo-driver/mongo/options""go.mongodb.org/mongo-driver/mongo/readpref""go.mongodb.org/mongo-driver/x/mongo/driver/auth"- "go.mongodb.org/mongo-driver/x/mongo/driver/topology""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -29,8 +29,9 @@ var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (defaultTimeout = 2 * time.Second// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.- keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)+ connStrPat = regexp.MustCompile(`\b(mongodb(?:\+srv)?://(?P<username>\S{3,50}):(?P<password>\S{3,88})@(?P<host>[-.%\w]+(?::\d{1,5})?(?:,[-.%\w]+(?::\d{1,5})?)*)(?:/(?P<authdb>[\w-]+)?(?P<options>\?\w+=[\w@/.$-]+(?:&(?:amp;)?\w+=[\w@/.$-]+)*)?)?)(?:\b|$)`)// TODO: Add support for sharded cluster, replica set and Atlas Deployment.+ placeholderPasswordPat = regexp.MustCompile(`^[xX]+|\*+$`))// Keywords are used for efficiently pre-filtering chunks.@@ -43,11 +44,17 @@ func (s Scanner) Keywords() []string {func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {dataStr := string(data)- matches := keyPat.FindAllStringSubmatch(dataStr, -1)+ matches := connStrPat.FindAllStringSubmatch(dataStr, -1)for _, match := range matches {- resMatch := strings.TrimSpace(match[1])+ // Filter out common placeholder passwords.+ password := match[3]+ if password == "" || placeholderPasswordPat.MatchString(password) {+ continue+ }+ // If the query string contains `&` the options will not be parsed.+ resMatch := strings.Replace(strings.TrimSpace(match[1]), "&", "&", -1)s1 := detectors.Result{DetectorType: detectorspb.DetectorType_MongoDB,Raw: []byte(resMatch),@@ -61,10 +68,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif timeout == 0 {timeout = defaultTimeout}- err := verifyUri(resMatch, timeout)- s1.Verified = err == nil- if !isErrDeterminate(err) {- s1.SetVerificationError(err, resMatch)+ isVerified, verificationErr := verifyUri(ctx, resMatch, timeout)+ s1.Verified = isVerified+ if !isErrDeterminate(verificationErr) {+ s1.SetVerificationError(verificationErr, resMatch)}}results = append(results, s1)@@ -78,23 +85,14 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {}func isErrDeterminate(err error) bool {- switch e := err.(type) {- case topology.ConnectionError:- switch e.Unwrap().(type) {- case *auth.Error:- return true- default:- return false- }- default:- return false- }+ var authErr *auth.Error+ return errors.As(err, &authErr)}-func verifyUri(uri string, timeout time.Duration) error {+func verifyUri(ctx context.Context, uri string, timeout time.Duration) (bool, error) {parsed, err := url.Parse(uri)if err != nil {- return err+ return false, err}params := url.Values{}@@ -114,16 +112,23 @@ func verifyUri(uri string, timeout time.Duration) error {parsed.Path = "/"uri = parsed.String()- ctx, cancel := context.WithTimeout(context.Background(), timeout)+ ctx, cancel := context.WithTimeout(ctx, timeout)defer cancel()- client, err := mongo.Connect(ctx, options.Client().SetTimeout(timeout).ApplyURI(uri))++ clientOptions := options.Client().SetTimeout(timeout).ApplyURI(uri)+ if err = clientOptions.Validate(); err != nil {+ return false, err+ }++ client, err := mongo.Connect(ctx, clientOptions)if err != nil {- return err+ return false, err}defer func() {_ = client.Disconnect(ctx)}()- return client.Ping(ctx, readpref.Primary())+ err = client.Ping(ctx, readpref.Primary())+ return err == nil, err}func (s Scanner) Type() detectorspb.DetectorType {diff --git a/pkg/detectors/mongodb/mongodb_test.go b/pkg/detectors/mongodb/mongodb_test.goindex c3074cb20..3b0d72dfc 100644--- a/pkg/detectors/mongodb/mongodb_test.go+++ b/pkg/detectors/mongodb/mongodb_test.go@@ -19,6 +19,201 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")+func TestMongoDB_Pattern(t *testing.T) {+ tests := []struct {+ name string+ data string+ shouldMatch bool+ match string+ }{+ // True positives+ {+ name: "long_password",+ data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`,+ shouldMatch: true,+ },+ {+ name: "long_password2",+ data: `mongodb://csb0230eada-2354-4c73-b3e4-8a1aaa996894:AiNtEyASbdXR5neJmTStMzKGItX2xvKuyEkcy65rviKD0ggZR19E1iVFIJ5ZAIY1xvvAiS5tOXsmACDbKDJIhQ==@csb0230eada-2354-4c73-b3e4-8a1aaa996894.mongo.cosmos.cloud-hostname.com:10255/csb-db0230eada-2354-4c73-b3e4-8a1aaa996894?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@csb0230eada-2354-4c73-b3e4-8a1aaa996894@`,+ shouldMatch: true,+ },+ {+ name: "long_password3",+ data: `mongodb://amsdfasfsadfdfdfpshot:6xNRRsdfsdfafd9NodO8vAFFBEHidfdfdfa87QDKXdCMubACDbhfQH1g==@amssdfafdafdadbsnapshot.mongo.cosmos.azure.com:10255/?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@amssadfasdfdbsnsdfadfapshot@`,+ shouldMatch: true,+ },+ {+ name: "single_host",+ data: `mongodb://myDBReader:D1fficultP%[email protected]`,+ shouldMatch: true,+ },+ {+ name: "single_host+port",+ data: `mongodb://myDBReader:D1fficultP%[email protected]:27017`,+ shouldMatch: true,+ },+ {+ name: "single_host+port+authdb",+ data: `mongodb://myDBReader:D1fficultP%[email protected]:27017/?authSource=admin`,+ shouldMatch: true,+ },+ {+ name: "single_host_ip",+ data: `mongodb://myDBReader:D1fficultP%[email protected]`,+ shouldMatch: true,+ },+ {+ name: "single_host_ip+port",+ data: `mongodb://myDBReader:D1fficultP%[email protected]:27017`,+ shouldMatch: true,+ },+ {+ name: "multiple_hosts_ip",+ data: `mongodb://root:[email protected]:27018,192.168.74.143:27019`,+ shouldMatch: true,+ },+ {+ name: "multiple_hosts_ip+slash",+ data: `mongodb://root:[email protected]:27018,192.168.74.143:27019/`,+ shouldMatch: true,+ },+ {+ name: "multiple_hosts+port+authdb",+ data: `mongodb://myDBReader:D1fficultP%[email protected]:27017,mongodb0.example.com:27017,mongodb0.example.com:27017/?authSource=admin`,+ shouldMatch: true,+ },+ {+ name: "multiple_hosts+options",+ data: `mongodb://username:[email protected]:27317,mongodb2.example.com,mongodb2.example.com:270/?connectTimeoutMS=300000&replicaSet=mySet&authSource=aDifferentAuthDB`,+ shouldMatch: true,+ },+ {+ name: "multiple_hosts2",+ data: `mongodb://prisma:[email protected]:27017,srv2.bu2lt.mongodb.net:27017,srv3.bu2lt.mongodb.net:27017/test?retryWrites=true&w=majority`,+ shouldMatch: true,+ },+ // TODO: These fail because the Go driver doesn't explicitly support `authMechanism=DEFAULT`[1].+ // However, this seems like a valid option[2] and I'm going to try to get that behaviour changed.+ //+ // [1] https://github.com/mongodb/mongo-go-driver/blob/master/x/mongo/driver/connstring/connstring.go#L450-L451+ // [2] https://www.mongodb.com/docs/drivers/node/current/fundamentals/authentication/mechanisms/+ {+ name: "encoded_options1",+ data: `mongodb://dave:password@localhost:27017/?authMechanism=DEFAULT&authSource=db&ssl=true"`,+ shouldMatch: true,+ match: "mongodb://dave:password@localhost:27017/?authMechanism=DEFAULT&authSource=db&ssl=true",+ },+ {+ name: "encoded_options2",+ data: `mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&sslverifycertificate=false`,+ shouldMatch: true,+ match: "mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&sslverifycertificate=false",+ },+ {+ name: "unix_socket",+ data: `mongodb://u%24ername:pa%24%24w%7B%7Drd@%2Ftmp%2Fmongodb-27017.sock/test`,+ shouldMatch: true,+ },+ {+ name: "dashes",+ data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`,+ shouldMatch: true,+ },+ {+ name: "protocol+srv",+ // TODO: Figure out how to handle `mongodb+srv`. It performs a DNS lookup, which fails if the host doesn't exist.+ //data: `mongodb+srv://root:[email protected]/mydb?retryWrites=true&w=majority`,+ data: `mongodb://root:[email protected]/mydb?retryWrites=true&w=majority`,+ shouldMatch: true,+ },+ {+ name: "0.0.0.0_host",+ data: `mongodb://username:[email protected]:27017/?authSource=admin`,+ shouldMatch: true,+ },+ {+ name: "localhost_host",+ data: `mongodb://username:password@localhost:27017/?authSource=admin`,+ shouldMatch: true,+ },+ {+ name: "127.0.0.1_host",+ data: `mongodb://username:[email protected]:27017/?authSource=admin`,+ shouldMatch: true,+ },+ {+ name: "docker_internal_host",+ data: `mongodb://username:[email protected]:27018/?authMechanism=PLAIN&tls=true&tlsCertificateKeyFile=/etc/certs/client.pem&tlsCaFile=/etc/certs/rootCA-cert.pem`,+ shouldMatch: true,+ },+ {+ name: "options_authsource_external",+ data: `mongodb://AKIAAAAAAAAAAAA:t9t2mawssecretkey@localhost:27017/?authMechanism=MONGODB-AWS&authsource=$external`,+ shouldMatch: true,+ },+ {+ name: "generic1",+ data: `mongodb://root:[email protected]:27017/`,+ shouldMatch: true,+ },++ // False positives+ {+ name: "no_password",+ data: `mongodb://mongodb0.example.com:27017/?replicaSet=myRepl`,+ shouldMatch: false,+ },+ {+ name: "empty",+ data: `mongodb://username:@mongodb0.example.com:27017/?replicaSet=myRepl`,+ shouldMatch: false,+ },+ {+ name: "placeholders_x+single_host",+ data: `mongodb://xxxx:xxxxx@xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`,+ shouldMatch: false,+ },+ {+ name: "placeholders_x+multiple_hosts",+ data: `mongodb://xxxx:xxxxx@xxxxxxx:3717,xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`,+ shouldMatch: false,+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ s := Scanner{}++ results, err := s.FromData(context.Background(), false, []byte(test.data))+ if err != nil {+ t.Errorf("MongoDB.FromData() error = %v", err)+ return+ }++ if test.shouldMatch {+ if len(results) == 0 {+ t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)+ return+ }+ expected := test.data+ if test.match != "" {+ expected = test.match+ }+ result := string(results[0].Raw)+ if result != expected {+ t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, result)+ return+ }+ } else {+ if len(results) > 0 {+ t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)+ return+ }+ }+ })+ }+}+func TestMongoDB_FromChunk(t *testing.T) {ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)defer cancel()