Skip to content

Find, verify, and analyze leaked credentials

[THOG-162] Implement JDBC verification for select drivers (#792)

Miccahddc81bd7c118
diff --git a/go.mod b/go.mod
index c421f6aac..d375f74b6 100644
--- a/go.mod
+++ b/go.mod
@@ -18,6 +18,7 @@ require (
github.com/bitfinexcom/bitfinex-api-go v0.0.0-20210608095005-9e0b26f200fb
github.com/bradleyfalzon/ghinstallation/v2 v2.1.0
github.com/crewjam/rfc5424 v0.1.0
+ github.com/denisenkom/go-mssqldb v0.12.2
github.com/envoyproxy/protoc-gen-validate v0.6.8
github.com/fatih/color v1.13.0
github.com/felixge/fgprof v0.9.3
@@ -26,6 +27,7 @@ require (
github.com/go-git/go-git/v5 v5.4.2
github.com/go-logr/logr v1.2.3
github.com/go-logr/zapr v1.2.3
+ github.com/go-sql-driver/mysql v1.6.0
github.com/golang-jwt/jwt v3.2.2+incompatible
github.com/google/go-cmp v0.5.9
github.com/google/go-github/v42 v42.0.0
@@ -35,7 +37,9 @@ require (
github.com/joho/godotenv v1.4.0
github.com/jpillora/overseer v1.1.6
github.com/kylelemons/godebug v1.1.0
+ github.com/lib/pq v1.10.7
github.com/mattn/go-colorable v0.1.13
+ github.com/mattn/go-sqlite3 v1.14.15
github.com/mholt/archiver/v4 v4.0.0-alpha.7
github.com/paulbellamy/ratecounter v0.2.0
github.com/pkg/errors v0.9.1
@@ -83,6 +87,8 @@ require (
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/go-ole/go-ole v1.2.6 // indirect
github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
+ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
+ github.com/golang-sql/sqlexp v0.1.0 // indirect
github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
@@ -128,6 +134,5 @@ require (
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/grpc v1.47.0 // indirect
gopkg.in/warnings.v0 v0.1.2 // indirect
- gopkg.in/yaml.v2 v2.4.0 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
diff --git a/go.sum b/go.sum
index 5db418b73..144fe35c5 100644
--- a/go.sum
+++ b/go.sum
@@ -62,6 +62,9 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
+github.com/Azure/azure-sdk-for-go/sdk/azcore v0.19.0/go.mod h1:h6H6c8enJmmocHUbLiiGY6sx7f9i+X3m1CHdd5c6Rdw=
+github.com/Azure/azure-sdk-for-go/sdk/azidentity v0.11.0/go.mod h1:HcM1YX14R7CJcghJGOYCgdezslRSVzqwLf/q+4Y2r/0=
+github.com/Azure/azure-sdk-for-go/sdk/internal v0.7.0/go.mod h1:yqy467j36fJxcRV2TzfVZ1pCb5vxm4BtZPUdYWe/Xo8=
github.com/Azure/go-autorest v14.2.0+incompatible h1:V5VMDjClD3GiElqLWO7mz2MxNAK/vTfRHdAubSIPRgs=
github.com/Azure/go-autorest v14.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
github.com/Azure/go-autorest/autorest v0.11.24 h1:1fIGgHKqVm54KIPT+q8Zmd1QlVsmHqeUGso5qm2BqqE=
@@ -138,8 +141,11 @@ github.com/crewjam/rfc5424 v0.1.0/go.mod h1:RCi9M3xHVOeerf6ULZzqv2xOGRO/zYaVUeRy
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/denisenkom/go-mssqldb v0.12.2 h1:1OcPn5GBIobjWNd+8yjfHNIaFX14B1pWI3F9HZy5KXw=
+github.com/denisenkom/go-mssqldb v0.12.2/go.mod h1:lnIw1mZukFRZDJYQ0Pb833QS2IaC3l5HkEfra2LJ+sk=
github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U=
github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE=
+github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ=
github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q=
github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo=
github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY=
@@ -188,6 +194,8 @@ github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=
github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=
github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=
+github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=
@@ -198,6 +206,10 @@ github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw
github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
github.com/golang-jwt/jwt/v4 v4.4.1 h1:pC5DB52sCeK48Wlb9oPcdhnjkz1TKt1D/P7WKJ0kUcQ=
github.com/golang-jwt/jwt/v4 v4.4.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZkZR4hgp4KJVfY3nMkvmwbVkpv1rVY=
+github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
+github.com/golang-sql/sqlexp v0.1.0 h1:ZCD6MBpcuOVfGVqsEmY5/4FtYiKz6tSyUv9LPEDei6A=
+github.com/golang-sql/sqlexp v0.1.0/go.mod h1:J4ad9Vo8ZCWQ2GMrC4UCQy1JpCbwU9m3EOqtpKwwwHI=
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -351,6 +363,8 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc=
github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw=
+github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
+github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
github.com/lyft/protoc-gen-star v0.6.1/go.mod h1:TGAoBVkt8w7MPG72TrKIu85MIdXwDuzJYeZuUPFPNwA=
github.com/matryer/is v1.2.0 h1:92UTHpy8CDwaJ08GqLDzhhuixiBUUD1p3AU6PHddz4A=
github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA=
@@ -362,10 +376,13 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky
github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
github.com/mattn/go-isatty v0.0.16 h1:bq3VjFmv/sOjHtdEhmkEV4x1AJtvUvOJ2PFAZ5+peKQ=
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
+github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/mholt/archiver/v4 v4.0.0-alpha.7 h1:xzByj8G8tj0Oq7ZYYU4+ixL/CVb5ruWCm0EZQ1PjOkE=
github.com/mholt/archiver/v4 v4.0.0-alpha.7/go.mod h1:Fs8qUkO74HHaidabihzYephJH8qmGD/nCP6tE5xC9BM=
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/modocache/gover v0.0.0-20171022184752-b58185e213c5/go.mod h1:caMODM3PzxT8aQXRPkAt8xlV/e7d7w8GM5g0fa5F0D8=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe h1:iruDEfMl2E6fbMZ9s0scYfZQ84/6SPL6zC8ACM2oIL0=
github.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32 h1:W6apQkHrMkS0Muv8G/TipAy/FJl/rCYT0+EuS8+Z0z4=
@@ -379,6 +396,7 @@ github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChl
github.com/pierrec/lz4/v4 v4.1.14 h1:+fL8AQEZtz/ijeNnpduH0bROTu0O3NZAlPjQxGn8LwE=
github.com/pierrec/lz4/v4 v4.1.14/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
+github.com/pkg/browser v0.0.0-20180916011732-0a3d74bf9ce4/go.mod h1:4OwLy04Bl9Ef3GJJCoec+30X3LQs/0/m4HFRt/2LUSA=
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 h1:+/+DxvQaYifJ+grD4klzrS5y+KJXldn/2YTl5JG+vZ8=
github.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7/go.mod h1:zO8QMzTeZd5cpnIkz/Gn6iK0jDfGicM1nynOkkPIl28=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -480,6 +498,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
@@ -562,6 +581,7 @@ golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLd
golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k=
golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210610132358-84b48f89b13b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
diff --git a/pkg/detectors/jdbc/jdbc.go b/pkg/detectors/jdbc/jdbc.go
index 83b0c32a9..b3ee1b348 100644
--- a/pkg/detectors/jdbc/jdbc.go
+++ b/pkg/detectors/jdbc/jdbc.go
@@ -2,8 +2,11 @@ package jdbc
import (
"context"
+ "database/sql"
+ "errors"
"regexp"
"strings"
+ "time"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -15,7 +18,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
- keyPat = regexp.MustCompile(`(?i)jdbc:[\w]{3,10}:\/\/\w[\s\S]{0,512}?password[=: \"']+(?P<pass>[^<{($]*?)[ \s'\"]+`)
+ keyPat = regexp.MustCompile(`(?i)jdbc:[\w]{3,10}:[^\s"']{0,512}`)
)
// Keywords are used for efficiently pre-filtering chunks.
@@ -30,38 +33,25 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
matches := keyPat.FindAllStringSubmatch(dataStr, -1)
for _, match := range matches {
- if match[1] == "" {
- continue
- }
- token := match[0]
- password := match[1]
-
- //TODO if username and password are the same, username will also be redacted... I think this is probably correct.
- redact := strings.TrimSpace(strings.Replace(token, password, strings.Repeat("*", len(password)), -1))
+ jdbcConn := match[0]
s := detectors.Result{
DetectorType: detectorspb.DetectorType_JDBC,
- Raw: []byte(token),
- Redacted: redact,
+ Raw: []byte(jdbcConn),
+ Redacted: tryRedactAnonymousJDBC(jdbcConn),
}
- //if verify {
- // // TODO: can this be verified? Possibly. Could triage verification to other DBMS strings
- // s.Verified = false
- // client := common.SaneHttpClient()
- // req, err := http.NewRequestWithContext(ctx, "GET", "https://jdbcci.com/api/v2/me", nil)
- // if err != nil {
- // continue
- // }
- // req.Header.Add("Accept", "application/json;")
- // req.Header.Add("Jdbc-Token", token)
- // res, err := client.Do(req)
- // if err == nil {
- // if res.StatusCode >= 200 && res.StatusCode < 300 {
- // s.Verified = true
- // }
- // }
- //}
+ if verify {
+ s.Verified = false
+ j, err := newJDBC(jdbcConn)
+ if err != nil {
+ continue
+ }
+ ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
+ defer cancel()
+ s.Verified = j.ping(ctx)
+ // TODO: specialized redaction
+ }
if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {
continue
@@ -72,3 +62,76 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return
}
+
+func tryRedactAnonymousJDBC(conn string) string {
+ userPass, postfix, found := strings.Cut(conn, "@")
+ if found {
+ if index := strings.LastIndex(userPass, ":"); index != -1 {
+ prefix, pass := userPass[:index], userPass[index+1:]
+ return prefix + ":" + strings.Repeat("*", len(pass)) + "@" + postfix
+ }
+ }
+ prefix, paramString, found := strings.Cut(conn, "?")
+ if !found {
+ return conn
+ }
+ var newParams []string
+ for _, param := range strings.Split(paramString, "&") {
+ key, val, _ := strings.Cut(param, "=")
+ if strings.Contains(strings.ToLower(key), "pass") {
+ newParams = append(newParams, key+"="+strings.Repeat("*", len(val)))
+ continue
+ }
+ newParams = append(newParams, param)
+ }
+ return prefix + "?" + strings.Join(newParams, "&")
+}
+
+var supportedSubprotocols = map[string]func(string) (jdbc, error){
+ "sqlite": parseSqlite,
+ "mysql": parseMySQL,
+ "postgresql": parsePostgres,
+ "sqlserver": parseSqlServer,
+}
+
+type jdbc interface {
+ ping(context.Context) bool
+}
+
+func newJDBC(conn string) (jdbc, error) {
+ // expected format: "jdbc:{subprotocol}:{subname}"
+ if !strings.HasPrefix(strings.ToLower(conn), "jdbc:") {
+ return nil, errors.New("expected jdbc prefix")
+ }
+ conn = conn[len("jdbc:"):]
+ subprotocol, subname, found := strings.Cut(conn, ":")
+ if !found {
+ return nil, errors.New("expected a colon separated subprotocol and subname")
+ }
+ // get the subprotocol parser
+ parser, ok := supportedSubprotocols[strings.ToLower(subprotocol)]
+ if !ok {
+ return nil, errors.New("unsupported subprotocol")
+ }
+ return parser(subname)
+}
+
+func ping(ctx context.Context, driverName, conn string) bool {
+ if err := pingErr(ctx, driverName, conn); err != nil {
+ return false
+ }
+ return true
+}
+
+func pingErr(ctx context.Context, driverName, conn string) error {
+ db, err := sql.Open(driverName, conn)
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+
+ if err := db.PingContext(ctx); err != nil {
+ return err
+ }
+ return nil
+}
diff --git a/pkg/detectors/jdbc/jdbc_integration_test.go b/pkg/detectors/jdbc/jdbc_integration_test.go
new file mode 100644
index 000000000..55c0bdf49
--- /dev/null
+++ b/pkg/detectors/jdbc/jdbc_integration_test.go
@@ -0,0 +1,127 @@
+//go:build detectors && integration
+// +build detectors,integration
+
+package jdbc
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "os/exec"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/kylelemons/godebug/pretty"
+ "github.com/stretchr/testify/assert"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
+)
+
+func TestMain(m *testing.M) {
+ code, err := runMain(m)
+ if err != nil {
+ panic(err)
+ }
+ os.Exit(code)
+}
+
+func runMain(m *testing.M) (int, error) {
+ if err := startPostgres(); err != nil {
+ return 0, err
+ }
+ defer stopPostgres()
+ if err := startMySQL(); err != nil {
+ return 0, err
+ }
+ defer stopMySQL()
+ return m.Run(), nil
+}
+
+func dockerLogLine(hash string, needle string) chan struct{} {
+ ch := make(chan struct{}, 1)
+ go func() {
+ for {
+ out, err := exec.Command("docker", "logs", hash).CombinedOutput()
+ if err != nil {
+ panic(err)
+ }
+ if strings.Contains(string(out), needle) {
+ ch <- struct{}{}
+ return
+ }
+ time.Sleep(1 * time.Second)
+ }
+ }()
+ return ch
+}
+
+func TestJdbcVerified(t *testing.T) {
+ type args struct {
+ ctx context.Context
+ data []byte
+ verify bool
+ }
+ tests := []struct {
+ name string
+ args args
+ want []detectors.Result
+ wantErr bool
+ }{
+ {
+ name: "postgres verified",
+ args: args{
+ ctx: context.Background(),
+ data: []byte(`jdbc connection string: jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=` + postgresPass),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_JDBC,
+ Verified: true,
+ Redacted: "jdbc:postgresql://localhost:5432/foo?sslmode=disable&password=" + strings.Repeat("*", len(postgresPass)),
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "mysql verified",
+ args: args{
+ ctx: context.Background(),
+ data: []byte(fmt.Sprintf(`CONN="jdbc:mysql://%s:%s@tcp(127.0.0.1:3306)/%s"`, mysqlUser, mysqlPass, mysqlDatabase)),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_JDBC,
+ Verified: true,
+ Redacted: fmt.Sprintf(`jdbc:mysql://%s:%s@tcp(127.0.0.1:3306)/%s`, mysqlUser, strings.Repeat("*", len(mysqlPass)), mysqlDatabase),
+ },
+ },
+ wantErr: false,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ s := Scanner{}
+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)
+ if tt.wantErr {
+ assert.Error(t, err)
+ return
+ }
+ assert.NoError(t, err)
+ if os.Getenv("FORCE_PASS_DIFF") == "true" {
+ return
+ }
+ for i := range got {
+ if len(got[i].Raw) == 0 {
+ t.Fatal("no raw secret present")
+ }
+ got[i].Raw = nil
+ }
+ if diff := pretty.Compare(got, tt.want); diff != "" {
+ t.Errorf("Jdbc.FromData() %s diff: (-got +want)\n%s", tt.name, diff)
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/jdbc/jdbc_test.go b/pkg/detectors/jdbc/jdbc_test.go
index fc2abff3f..76a33d181 100644
--- a/pkg/detectors/jdbc/jdbc_test.go
+++ b/pkg/detectors/jdbc/jdbc_test.go
@@ -21,18 +21,16 @@ func TestJdbc_FromChunk(t *testing.T) {
}
tests := []struct {
name string
- s Scanner
args args
want []detectors.Result
wantErr bool
}{
{
name: "found, unverified",
- s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(`jdbc connection string: jdbc:mysql://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=testpassword <-`),
- verify: true,
+ verify: false,
},
want: []detectors.Result{
{
@@ -45,7 +43,6 @@ func TestJdbc_FromChunk(t *testing.T) {
},
{
name: "found, unverified numeric password",
- s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte(`jdbc connection string: jdbc:postgresql://host:5342/testdb?password=123456 <-`),
@@ -62,15 +59,62 @@ func TestJdbc_FromChunk(t *testing.T) {
},
{
name: "not found",
- s: Scanner{},
args: args{
ctx: context.Background(),
data: []byte("You cannot find the secret within"),
- verify: true,
+ verify: false,
},
want: nil,
wantErr: false,
},
+ {
+ name: "sqlite unverified",
+ args: args{
+ ctx: context.Background(),
+ data: []byte("jdbc:sqlite::memory:"),
+ verify: true,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_JDBC,
+ Verified: false,
+ Redacted: "jdbc:sqlite::memory:",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found double quoted string, unverified",
+ args: args{
+ ctx: context.Background(),
+ data: []byte(`CONN="jdbc:postgres://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=testpassword"`),
+ verify: false,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_JDBC,
+ Verified: false,
+ Redacted: "jdbc:postgres://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=************",
+ },
+ },
+ wantErr: false,
+ },
+ {
+ name: "found single quoted string, unverified",
+ args: args{
+ ctx: context.Background(),
+ data: []byte(`CONN='jdbc:postgres://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=testpassword'`),
+ verify: false,
+ },
+ want: []detectors.Result{
+ {
+ DetectorType: detectorspb.DetectorType_JDBC,
+ Verified: false,
+ Redacted: "jdbc:postgres://hello.test.us-east-1.rds.amazonaws.com:3306/testdb?password=************",
+ },
+ },
+ wantErr: false,
+ },
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
diff --git a/pkg/detectors/jdbc/mysql.go b/pkg/detectors/jdbc/mysql.go
new file mode 100644
index 000000000..768d1ed2a
--- /dev/null
+++ b/pkg/detectors/jdbc/mysql.go
@@ -0,0 +1,66 @@
+package jdbc
+
+import (
+ "context"
+ "errors"
+ "strings"
+
+ _ "github.com/go-sql-driver/mysql"
+)
+
+type mysqlJDBC struct {
+ conn string
+ userPass string
+ host string
+ database string
+ params string
+}
+
+func (s *mysqlJDBC) ping(ctx context.Context) bool {
+ if ping(ctx, "mysql", s.conn) {
+ return true
+ }
+ // try building connection string (should be same as s.conn though)
+ if ping(ctx, "mysql", s.build()) {
+ return true
+ }
+ // try removing database
+ s.database = ""
+ return ping(ctx, "mysql", s.build())
+}
+
+func (s *mysqlJDBC) build() string {
+ conn := s.host + "/" + s.database
+ if s.userPass != "" {
+ conn = s.userPass + "@" + conn
+ }
+ if s.params != "" {
+ conn = conn + "?" + s.params
+ }
+ return conn
+}
+
+func parseMySQL(subname string) (jdbc, error) {
+ // expected form: [subprotocol:]//[user:password@]HOST[/DB][?key=val[&key=val]]
+ hostAndDB, params, _ := strings.Cut(subname, "?")
+ if !strings.HasPrefix(hostAndDB, "//") {
+ return nil, errors.New("expected host to start with //")
+ }
+ userPassAndHostAndDB := strings.TrimPrefix(hostAndDB, "//")
+ userPass, hostAndDB, found := strings.Cut(userPassAndHostAndDB, "@")
+ if !found {
+ hostAndDB = userPass
+ userPass = ""
+ }
+ host, database, found := strings.Cut(hostAndDB, "/")
+ if !found {
+ return nil, errors.New("expected host and database to be separated by /")
+ }
+ return &mysqlJDBC{
+ conn: subname[2:],
+ userPass: userPass,
+ host: host,
+ database: database,
+ params: params,
+ }, nil
+}
diff --git a/pkg/detectors/jdbc/mysql_integration_test.go b/pkg/detectors/jdbc/mysql_integration_test.go
new file mode 100644
index 000000000..b5d503c7a
--- /dev/null
+++ b/pkg/detectors/jdbc/mysql_integration_test.go
@@ -0,0 +1,95 @@
+//go:build detectors && integration
+// +build detectors,integration
+
+package jdbc
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "os/exec"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const (
+ mysqlUser = "coolGuy"
+ mysqlPass = "23201dabb56ca236f3dc6736c0f9afad"
+ mysqlDatabase = "stuff"
+)
+
+func TestMySQL(t *testing.T) {
+ tests := []struct {
+ input string
+ wantErr bool
+ wantPing bool
+ }{
+ {
+ input: "",
+ wantErr: true,
+ },
+ {
+ input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+ wantPing: true,
+ },
+ {
+ input: "//wrongUser:wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+ wantPing: false,
+ },
+ {
+ input: "//" + mysqlUser + ":wrongPass@tcp(127.0.0.1:3306)/" + mysqlDatabase,
+ wantPing: false,
+ },
+ {
+ input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/",
+ wantPing: true,
+ },
+ {
+ input: "//" + mysqlUser + ":" + mysqlPass + "@tcp(127.0.0.1:3306)/wrongDB",
+ wantPing: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.input, func(t *testing.T) {
+ j, err := parseMySQL(tt.input)
+ if tt.wantErr {
+ assert.Error(t, err)
+ return
+ }
+ assert.NoError(t, err)
+ assert.Equal(t, tt.wantPing, j.ping(context.Background()))
+ })
+ }
+}
+
+var mysqlDockerHash string
+
+func startMySQL() error {
+ cmd := exec.Command(
+ "docker", "run", "--rm", "-p", "3306:3306",
+ "-e", "MYSQL_ROOT_PASSWORD=403a96cff2a323f74bfb1c16992895be",
+ "-e", "MYSQL_USER="+mysqlUser,
+ "-e", "MYSQL_PASSWORD="+mysqlPass,
+ "-e", "MYSQL_DATABASE="+mysqlDatabase,
+ "-e", "MYSQL_ROOT_HOST=%",
+ "-d", "mysql",
+ )
+ out, err := cmd.Output()
+ if err != nil {
+ return err
+ }
+ mysqlDockerHash = string(bytes.TrimSpace(out))
+ select {
+ case <-dockerLogLine(mysqlDockerHash, "socket: '/var/run/mysqld/mysqld.sock' port: 3306"):
+ return nil
+ case <-time.After(30 * time.Second):
+ stopMySQL()
+ return errors.New("timeout waiting for mysql database to be ready")
+ }
+}
+
+func stopMySQL() {
+ exec.Command("docker", "kill", mysqlDockerHash).Run()
+}
diff --git a/pkg/detectors/jdbc/postgres.go b/pkg/detectors/jdbc/postgres.go
new file mode 100644
index 000000000..163e14d09
--- /dev/null
+++ b/pkg/detectors/jdbc/postgres.go
@@ -0,0 +1,81 @@
+package jdbc
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+
+ _ "github.com/lib/pq"
+)
+
+type postgresJDBC struct {
+ conn string
+ params map[string]string
+}
+
+func (s *postgresJDBC) ping(ctx context.Context) bool {
+ // try the provided connection string directly
+ if ping(ctx, "postgres", s.conn) {
+ return true
+ }
+ // try as a URL
+ if ping(ctx, "postgres", "postgres://"+s.conn) {
+ return true
+ }
+ // build a connection string
+ data := map[string]string{
+ // default user
+ "user": "postgres",
+ }
+ for key, val := range s.params {
+ if key == "host" {
+ if h, p, found := strings.Cut(val, ":"); found {
+ data["host"] = h
+ data["port"] = p
+ continue
+ }
+ }
+ data[key] = val
+ }
+ if ping(ctx, "postgres", joinKeyValues(data)) {
+ return true
+ }
+ if s.params["dbname"] != "" {
+ delete(s.params, "dbname")
+ return s.ping(ctx)
+ }
+ return false
+}
+
+func joinKeyValues(m map[string]string) string {
+ var data []string
+ for k, v := range m {
+ if v == "" {
+ continue
+ }
+ data = append(data, fmt.Sprintf("%s=%s", k, v))
+ }
+ return strings.Join(data, " ")
+}
+
+func parsePostgres(subname string) (jdbc, error) {
+ // expected form: //HOST/DB?key=value&key=value
+ hostAndDB, paramString, _ := strings.Cut(subname, "?")
+ if !strings.HasPrefix(hostAndDB, "//") {
+ return nil, errors.New("expected host to start with //")
+ }
+ hostAndDB = strings.TrimPrefix(hostAndDB, "//")
+ host, database, _ := strings.Cut(hostAndDB, "/")
+
+ params := map[string]string{
+ "host": host,
+ "dbname": database,
+ }
+ for _, param := range strings.Split(paramString, "&") {
+ key, val, _ := strings.Cut(param, "=")
+ params[key] = val
+ }
+
+ return &postgresJDBC{subname[2:], params}, nil
+}
diff --git a/pkg/detectors/jdbc/postgres_integration_test.go b/pkg/detectors/jdbc/postgres_integration_test.go
new file mode 100644
index 000000000..fb4fe93b6
--- /dev/null
+++ b/pkg/detectors/jdbc/postgres_integration_test.go
@@ -0,0 +1,95 @@
+//go:build detectors && integration
+// +build detectors,integration
+
+package jdbc
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "os/exec"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/assert"
+)
+
+const (
+ postgresUser = "postgres"
+ postgresPass = "23201dabb56ca236f3dc6736c0f9afad"
+)
+
+func TestPostgres(t *testing.T) {
+ tests := []struct {
+ input string
+ wantErr bool
+ wantPing bool
+ }{
+ {
+ input: "//localhost:5432/foo?sslmode=disable&password=" + postgresPass,
+ wantPing: true,
+ },
+ {
+ input: "//localhost:5432/foo?sslmode=disable&user=" + postgresUser + "&password=" + postgresPass,
+ wantPing: true,
+ },
+ {
+ input: "//localhost/foo?sslmode=disable&port=5432&password=" + postgresPass,
+ wantPing: true,
+ },
+ {
+ input: "//localhost:5432/foo?password=" + postgresPass,
+ wantPing: false,
+ },
+ {
+ input: "//localhost:5432/foo?sslmode=disable&password=foo",
+ wantPing: false,
+ },
+ {
+ input: "//localhost:5432/foo?sslmode=disable&user=foo&password=" + postgresPass,
+ wantPing: false,
+ },
+ {
+ input: "invalid",
+ wantErr: true,
+ },
+ }
+ for _, tt := range tests {
+ t.Run(tt.input, func(t *testing.T) {
+ j, err := parsePostgres(tt.input)
+ if tt.wantErr {
+ assert.Error(t, err)
+ return
+ }
+ assert.NoError(t, err)
+ assert.Equal(t, tt.wantPing, j.ping(context.Background()))
+ })
+ }
+}
+
+var postgresDockerHash string
+
+func startPostgres() error {
+ cmd := exec.Command(
+ "docker", "run", "--rm", "-p", "5432:5432",
+ "-e", "POSTGRES_PASSWORD="+postgresPass,
+ "-e", "POSTGRES_USER="+postgresUser,
+ "-d", "postgres",
+ )
+ out, err := cmd.Output()
+ if err != nil {
+ return err
+ }
+ postgresDockerHash = string(bytes.TrimSpace(out))
+ select {
+ case <-dockerLogLine(postgresDockerHash, "PostgreSQL init process complete; ready for start up."):
+ return nil
+ case <-time.After(30 * time.Second):
+ stopPostgres()
+ return errors.New("timeout waiting for postgres database to be ready")
+ }
+}
+
+func stopPostgres() {
+ exec.Command("docker", "kill", postgresDockerHash).Run()
+}
diff --git a/pkg/detectors/jdbc/sqlite.go b/pkg/detectors/jdbc/sqlite.go
new file mode 100644
index 000000000..edf8aa867
--- /dev/null
+++ b/pkg/detectors/jdbc/sqlite.go
@@ -0,0 +1,37 @@
+package jdbc
+
+import (
+ "context"
+ "errors"
+ "strings"
+
+ _ "github.com/mattn/go-sqlite3"
+)
+
+type sqliteJDBC struct {
+ filename string
+ params map[string]string
+ testing bool
+}
+
+func (s *sqliteJDBC) ping(ctx context.Context) bool {
+ if !s.testing {
+ // sqlite is not a networked database, so we cannot verify
+ return false
+ }
+ return ping(ctx, "sqlite3", s.filename)
+}
+
+func parseSqlite(subname string) (jdbc, error) {
+ filename, params, _ := strings.Cut(subname, "?")
+ if filename == "" {
+ return nil, errors.New("empty filename")
+ }
+ j := &sqliteJDBC{filename: filename, params: map[string]string{}}
+ for _, keyVal := range strings.Split(params, "&") {
+ if key, val, found := strings.Cut(keyVal, "="); found {
+ j.params[key] = val
+ }
+ }
+ return j, nil
+}
diff --git a/pkg/detectors/jdbc/sqlite_test.go b/pkg/detectors/jdbc/sqlite_test.go
new file mode 100644
index 000000000..4e2578bb3
--- /dev/null
+++ b/pkg/detectors/jdbc/sqlite_test.go
@@ -0,0 +1,46 @@
+//go:build detectors
+// +build detectors
+
+package jdbc
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func parseSqliteTest(subname string) (jdbc, error) {
+ j, err := parseSqlite(subname)
+ if err != nil {
+ return j, err
+ }
+ j.(*sqliteJDBC).testing = true
+ return j, err
+}
+
+func TestParseSqlite(t *testing.T) {
+ type testStruct struct {
+ input string
+ wantErr bool
+ }
+ tests := []struct {
+ input string
+ wantErr bool
+ }{
+ {input: "", wantErr: true},
+ {input: ":memory:"},
+ }
+
+ for _, test := range tests {
+ t.Run(test.input, func(t *testing.T) {
+ j, err := parseSqliteTest(test.input)
+ if test.wantErr {
+ assert.Error(t, err)
+ } else {
+ assert.NoError(t, err)
+ assert.True(t, j.ping(context.Background()))
+ }
+ })
+ }
+}
diff --git a/pkg/detectors/jdbc/sqlserver.go b/pkg/detectors/jdbc/sqlserver.go
new file mode 100644
index 000000000..3f545a3b7
--- /dev/null
+++ b/pkg/detectors/jdbc/sqlserver.go
@@ -0,0 +1,25 @@
+package jdbc
+
+import (
+ "context"
+ "strings"
+
+ _ "github.com/denisenkom/go-mssqldb"
+)
+
+type sqlServerJDBC struct {
+ conn string
+}
+
+func (s *sqlServerJDBC) ping(ctx context.Context) bool {
+ if ping(ctx, "mssql", s.conn) {
+ return true
+ }
+ // try URL format
+ return ping(ctx, "mssql", "sqlserver://"+s.conn)
+}
+
+func parseSqlServer(subname string) (jdbc, error) {
+ // expected form: //[username:password@]host/instance[?key=val[&key=val]]
+ return &sqlServerJDBC{strings.TrimPrefix(subname, "//")}, nil
+}