Skip to content

Find, verify, and analyze leaked credentials

Improve MongoDB connection string matching (#1550)

Richard Gomez75557f61edd5
diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.go
index 8593fae01..22bced1fa 100644
--- a/pkg/detectors/mongodb/mongodb.go
+++ b/pkg/detectors/mongodb/mongodb.go
@@ -2,6 +2,7 @@ package mongodb
import (
"context"
+ "errors"
"net/url"
"strings"
"time"
@@ -12,7 +13,6 @@ import (
"go.mongodb.org/mongo-driver/mongo/options"
"go.mongodb.org/mongo-driver/mongo/readpref"
"go.mongodb.org/mongo-driver/x/mongo/driver/auth"
- "go.mongodb.org/mongo-driver/x/mongo/driver/topology"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -29,8 +29,9 @@ var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
var (
defaultTimeout = 2 * time.Second
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
- keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,88})@[-.%\w\/:]+)\b`)
+ connStrPat = regexp.MustCompile(`\b(mongodb(?:\+srv)?://(?P<username>\S{3,50}):(?P<password>\S{3,88})@(?P<host>[-.%\w]+(?::\d{1,5})?(?:,[-.%\w]+(?::\d{1,5})?)*)(?:/(?P<authdb>[\w-]+)?(?P<options>\?\w+=[\w@/.$-]+(?:&(?:amp;)?\w+=[\w@/.$-]+)*)?)?)(?:\b|$)`)
// TODO: Add support for sharded cluster, replica set and Atlas Deployment.
+ placeholderPasswordPat = regexp.MustCompile(`^[xX]+|\*+$`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -43,11 +44,17 @@ func (s Scanner) Keywords() []string {
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
dataStr := string(data)
- matches := keyPat.FindAllStringSubmatch(dataStr, -1)
+ matches := connStrPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
- resMatch := strings.TrimSpace(match[1])
+ // Filter out common placeholder passwords.
+ password := match[3]
+ if password == "" || placeholderPasswordPat.MatchString(password) {
+ continue
+ }
+ // If the query string contains `&amp;` the options will not be parsed.
+ resMatch := strings.Replace(strings.TrimSpace(match[1]), "&amp;", "&", -1)
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_MongoDB,
Raw: []byte(resMatch),
@@ -61,10 +68,10 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if timeout == 0 {
timeout = defaultTimeout
}
- err := verifyUri(resMatch, timeout)
- s1.Verified = err == nil
- if !isErrDeterminate(err) {
- s1.SetVerificationError(err, resMatch)
+ isVerified, verificationErr := verifyUri(ctx, resMatch, timeout)
+ s1.Verified = isVerified
+ if !isErrDeterminate(verificationErr) {
+ s1.SetVerificationError(verificationErr, resMatch)
}
}
results = append(results, s1)
@@ -78,23 +85,14 @@ func (s Scanner) IsFalsePositive(_ detectors.Result) (bool, string) {
}
func isErrDeterminate(err error) bool {
- switch e := err.(type) {
- case topology.ConnectionError:
- switch e.Unwrap().(type) {
- case *auth.Error:
- return true
- default:
- return false
- }
- default:
- return false
- }
+ var authErr *auth.Error
+ return errors.As(err, &authErr)
}
-func verifyUri(uri string, timeout time.Duration) error {
+func verifyUri(ctx context.Context, uri string, timeout time.Duration) (bool, error) {
parsed, err := url.Parse(uri)
if err != nil {
- return err
+ return false, err
}
params := url.Values{}
@@ -114,16 +112,23 @@ func verifyUri(uri string, timeout time.Duration) error {
parsed.Path = "/"
uri = parsed.String()
- ctx, cancel := context.WithTimeout(context.Background(), timeout)
+ ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()
- client, err := mongo.Connect(ctx, options.Client().SetTimeout(timeout).ApplyURI(uri))
+
+ clientOptions := options.Client().SetTimeout(timeout).ApplyURI(uri)
+ if err = clientOptions.Validate(); err != nil {
+ return false, err
+ }
+
+ client, err := mongo.Connect(ctx, clientOptions)
if err != nil {
- return err
+ return false, err
}
defer func() {
_ = client.Disconnect(ctx)
}()
- return client.Ping(ctx, readpref.Primary())
+ err = client.Ping(ctx, readpref.Primary())
+ return err == nil, err
}
func (s Scanner) Type() detectorspb.DetectorType {
diff --git a/pkg/detectors/mongodb/mongodb_test.go b/pkg/detectors/mongodb/mongodb_test.go
index c3074cb20..3b0d72dfc 100644
--- a/pkg/detectors/mongodb/mongodb_test.go
+++ b/pkg/detectors/mongodb/mongodb_test.go
@@ -19,6 +19,201 @@ import (
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
+func TestMongoDB_Pattern(t *testing.T) {
+ tests := []struct {
+ name string
+ data string
+ shouldMatch bool
+ match string
+ }{
+ // True positives
+ {
+ name: "long_password",
+ data: `mongodb://agenda-live:m21w7PFfRXQwfHZU1Fgx0rTX29ZBQaWMODLeAjsmyslVcMmcmy6CnLyu3byVDtdLYcCokze8lIE4KyAgSCGZxQ==@agenda-live.mongo.cosmos.azure.com:10255/?retryWrites=false&ssl=true&replicaSet=globaldb&maxIdleTimeMS=120000&appName=@agenda-live@`,
+ shouldMatch: true,
+ },
+ {
+ name: "long_password2",
+ data: `mongodb://csb0230eada-2354-4c73-b3e4-8a1aaa996894:AiNtEyASbdXR5neJmTStMzKGItX2xvKuyEkcy65rviKD0ggZR19E1iVFIJ5ZAIY1xvvAiS5tOXsmACDbKDJIhQ==@csb0230eada-2354-4c73-b3e4-8a1aaa996894.mongo.cosmos.cloud-hostname.com:10255/csb-db0230eada-2354-4c73-b3e4-8a1aaa996894?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@csb0230eada-2354-4c73-b3e4-8a1aaa996894@`,
+ shouldMatch: true,
+ },
+ {
+ name: "long_password3",
+ data: `mongodb://amsdfasfsadfdfdfpshot:6xNRRsdfsdfafd9NodO8vAFFBEHidfdfdfa87QDKXdCMubACDbhfQH1g==@amssdfafdafdadbsnapshot.mongo.cosmos.azure.com:10255/?ssl=true&replicaSet=globaldb&retrywrites=false&maxIdleTimeMS=120000&appName=@amssadfasdfdbsnsdfadfapshot@`,
+ shouldMatch: true,
+ },
+ {
+ name: "single_host",
+ data: `mongodb://myDBReader:D1fficultP%[email protected]`,
+ shouldMatch: true,
+ },
+ {
+ name: "single_host+port",
+ data: `mongodb://myDBReader:D1fficultP%[email protected]:27017`,
+ shouldMatch: true,
+ },
+ {
+ name: "single_host+port+authdb",
+ data: `mongodb://myDBReader:D1fficultP%[email protected]:27017/?authSource=admin`,
+ shouldMatch: true,
+ },
+ {
+ name: "single_host_ip",
+ data: `mongodb://myDBReader:D1fficultP%[email protected]`,
+ shouldMatch: true,
+ },
+ {
+ name: "single_host_ip+port",
+ data: `mongodb://myDBReader:D1fficultP%[email protected]:27017`,
+ shouldMatch: true,
+ },
+ {
+ name: "multiple_hosts_ip",
+ data: `mongodb://root:[email protected]:27018,192.168.74.143:27019`,
+ shouldMatch: true,
+ },
+ {
+ name: "multiple_hosts_ip+slash",
+ data: `mongodb://root:[email protected]:27018,192.168.74.143:27019/`,
+ shouldMatch: true,
+ },
+ {
+ name: "multiple_hosts+port+authdb",
+ data: `mongodb://myDBReader:D1fficultP%[email protected]:27017,mongodb0.example.com:27017,mongodb0.example.com:27017/?authSource=admin`,
+ shouldMatch: true,
+ },
+ {
+ name: "multiple_hosts+options",
+ data: `mongodb://username:[email protected]:27317,mongodb2.example.com,mongodb2.example.com:270/?connectTimeoutMS=300000&replicaSet=mySet&authSource=aDifferentAuthDB`,
+ shouldMatch: true,
+ },
+ {
+ name: "multiple_hosts2",
+ data: `mongodb://prisma:[email protected]:27017,srv2.bu2lt.mongodb.net:27017,srv3.bu2lt.mongodb.net:27017/test?retryWrites=true&w=majority`,
+ shouldMatch: true,
+ },
+ // TODO: These fail because the Go driver doesn't explicitly support `authMechanism=DEFAULT`[1].
+ // However, this seems like a valid option[2] and I'm going to try to get that behaviour changed.
+ //
+ // [1] https://github.com/mongodb/mongo-go-driver/blob/master/x/mongo/driver/connstring/connstring.go#L450-L451
+ // [2] https://www.mongodb.com/docs/drivers/node/current/fundamentals/authentication/mechanisms/
+ {
+ name: "encoded_options1",
+ data: `mongodb://dave:password@localhost:27017/?authMechanism=DEFAULT&amp;authSource=db&amp;ssl=true&quot;`,
+ shouldMatch: true,
+ match: "mongodb://dave:password@localhost:27017/?authMechanism=DEFAULT&authSource=db&ssl=true",
+ },
+ {
+ name: "encoded_options2",
+ data: `mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&amp;sslverifycertificate=false`,
+ shouldMatch: true,
+ match: "mongodb://cefapp:MdTc8Kc8DzlTE1RUl1JVDGS4zw1U1t6145sPWqeStWA50xEUKPfUCGlnk3ACkfqH6qLAwpnm9awpY1m8dg0YlQ==@cefapp.documents.azure.com:10250/?ssl=true&sslverifycertificate=false",
+ },
+ {
+ name: "unix_socket",
+ data: `mongodb://u%24ername:pa%24%24w%7B%7Drd@%2Ftmp%2Fmongodb-27017.sock/test`,
+ shouldMatch: true,
+ },
+ {
+ name: "dashes",
+ data: `mongodb://db-user:db-password@mongodb-instance:27017/db-name`,
+ shouldMatch: true,
+ },
+ {
+ name: "protocol+srv",
+ // TODO: Figure out how to handle `mongodb+srv`. It performs a DNS lookup, which fails if the host doesn't exist.
+ //data: `mongodb+srv://root:[email protected]/mydb?retryWrites=true&w=majority`,
+ data: `mongodb://root:[email protected]/mydb?retryWrites=true&w=majority`,
+ shouldMatch: true,
+ },
+ {
+ name: "0.0.0.0_host",
+ data: `mongodb://username:[email protected]:27017/?authSource=admin`,
+ shouldMatch: true,
+ },
+ {
+ name: "localhost_host",
+ data: `mongodb://username:password@localhost:27017/?authSource=admin`,
+ shouldMatch: true,
+ },
+ {
+ name: "127.0.0.1_host",
+ data: `mongodb://username:[email protected]:27017/?authSource=admin`,
+ shouldMatch: true,
+ },
+ {
+ name: "docker_internal_host",
+ data: `mongodb://username:[email protected]:27018/?authMechanism=PLAIN&tls=true&tlsCertificateKeyFile=/etc/certs/client.pem&tlsCaFile=/etc/certs/rootCA-cert.pem`,
+ shouldMatch: true,
+ },
+ {
+ name: "options_authsource_external",
+ data: `mongodb://AKIAAAAAAAAAAAA:t9t2mawssecretkey@localhost:27017/?authMechanism=MONGODB-AWS&authsource=$external`,
+ shouldMatch: true,
+ },
+ {
+ name: "generic1",
+ data: `mongodb://root:[email protected]:27017/`,
+ shouldMatch: true,
+ },
+
+ // False positives
+ {
+ name: "no_password",
+ data: `mongodb://mongodb0.example.com:27017/?replicaSet=myRepl`,
+ shouldMatch: false,
+ },
+ {
+ name: "empty",
+ data: `mongodb://username:@mongodb0.example.com:27017/?replicaSet=myRepl`,
+ shouldMatch: false,
+ },
+ {
+ name: "placeholders_x+single_host",
+ data: `mongodb://xxxx:xxxxx@xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`,
+ shouldMatch: false,
+ },
+ {
+ name: "placeholders_x+multiple_hosts",
+ data: `mongodb://xxxx:xxxxx@xxxxxxx:3717,xxxxxxx:3717/zkquant?replicaSet=mgset-3017917`,
+ shouldMatch: false,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ s := Scanner{}
+
+ results, err := s.FromData(context.Background(), false, []byte(test.data))
+ if err != nil {
+ t.Errorf("MongoDB.FromData() error = %v", err)
+ return
+ }
+
+ if test.shouldMatch {
+ if len(results) == 0 {
+ t.Errorf("%s: did not receive a match for '%v' when one was expected", test.name, test.data)
+ return
+ }
+ expected := test.data
+ if test.match != "" {
+ expected = test.match
+ }
+ result := string(results[0].Raw)
+ if result != expected {
+ t.Errorf("%s: did not receive expected match.\n\texpected: '%s'\n\t actual: '%s'", test.name, expected, result)
+ return
+ }
+ } else {
+ if len(results) > 0 {
+ t.Errorf("%s: received a match for '%v' when one wasn't wanted", test.name, test.data)
+ return
+ }
+ }
+ })
+ }
+}
+
func TestMongoDB_FromChunk(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
defer cancel()