Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Do local URI verification, while attempting to defuse SSRF (#879)
Dustin Deckera7fc12240f8c
diff --git a/go.mod b/go.modindex 36067b2b7..75d44b126 100644--- a/go.mod+++ b/go.mod@@ -27,6 +27,7 @@ require (github.com/go-git/go-git/v5 v5.4.2github.com/go-logr/logr v1.2.3github.com/go-logr/zapr v1.2.3+ github.com/go-redis/redis v6.15.9+incompatiblegithub.com/go-sql-driver/mysql v1.6.0github.com/golang-jwt/jwt v3.2.2+incompatiblegithub.com/google/go-cmp v0.5.9@@ -34,6 +35,7 @@ require (github.com/gorilla/mux v1.8.0github.com/h2non/filetype v1.1.3github.com/hashicorp/go-retryablehttp v0.7.1+ github.com/jlaffaye/ftp v0.1.0github.com/joho/godotenv v1.4.0github.com/jpillora/overseer v1.1.6github.com/kylelemons/godebug v1.1.0@@ -97,7 +99,9 @@ require (github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirectgithub.com/googleapis/gax-go/v2 v2.6.0 // indirectgithub.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect+ github.com/hashicorp/errwrap v1.0.0 // indirectgithub.com/hashicorp/go-cleanhttp v0.5.2 // indirect+ github.com/hashicorp/go-multierror v1.1.1 // indirectgithub.com/imdario/mergo v0.3.12 // indirectgithub.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirectgithub.com/jmespath/go-jmespath v0.4.0 // indirect@@ -109,6 +113,8 @@ require (github.com/mitchellh/go-homedir v1.1.0 // indirectgithub.com/montanaflynn/stats v0.0.0-20171201202039-1bf9dbcd8cbe // indirectgithub.com/nwaples/rardecode/v2 v2.0.0-beta.2 // indirect+ github.com/onsi/ginkgo v1.16.5 // indirect+ github.com/onsi/gomega v1.23.0 // indirectgithub.com/pierrec/lz4/v4 v4.1.14 // indirectgithub.com/pkg/diff v0.0.0-20200914180035-5b29258ca4f7 // indirectgithub.com/pmezard/go-difflib v1.0.0 // indirect@@ -124,9 +130,9 @@ require (go.uber.org/atomic v1.7.0 // indirectgo.uber.org/multierr v1.6.0 // indirectgolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 // indirect- golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 // indirect- golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec // indirect- golang.org/x/text v0.3.7 // indirect+ golang.org/x/net v0.1.0 // indirect+ golang.org/x/sys v0.1.0 // indirect+ golang.org/x/text v0.4.0 // indirectgolang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirectgolang.org/x/tools v0.1.12 // indirectgoogle.golang.org/api v0.99.0 // indirectdiff --git a/go.sum b/go.sumindex 21087e34a..57a5ee069 100644--- a/go.sum+++ b/go.sum@@ -135,6 +135,9 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFgithub.com/felixge/fgprof v0.9.3 h1:VvyZxILNuCiUCSXtPtYmmtGvb65nqXh2QFWc0Wpf2/g=github.com/felixge/fgprof v0.9.3/go.mod h1:RdbpDgzqYVh/T9fPELJyV7EYJuHB55UTEULNun8eiPw=github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=+github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=+github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4=+github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=github.com/getsentry/sentry-go v0.14.0 h1:rlOBkuFZRKKdUnKO+0U3JclRDQKlRu5vVQtkWSQvC70=github.com/getsentry/sentry-go v0.14.0/go.mod h1:RZPJKSw+adu8PBNygiri/A98FqVr2HtRckJk9XVxJ9I=github.com/gliderlabs/ssh v0.2.2 h1:6zsha5zo/TWhRhwqCD3+EarCAgZ2yN28ipRnGPnwkI0=@@ -160,8 +163,11 @@ github.com/go-logr/zapr v1.2.3 h1:a9vnzlIBPQBBkeaR9IuMUfmVOrQlkoC4YfPoFkX3T7A=github.com/go-logr/zapr v1.2.3/go.mod h1:eIauM6P8qSvTw5o2ez6UEAfGjQKrxQTl5EoK+Qa2oG4=github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=+github.com/go-redis/redis v6.15.9+incompatible h1:K0pv1D7EQUjfyoMql+r/jZqCLizCGKFlFgcHWWmHQjg=+github.com/go-redis/redis v6.15.9+incompatible/go.mod h1:NAIEuMOZ/fxfXJIrKDQDz8wamY7mA7PouImQ2Jvg6kA=github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE=github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=+github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=github.com/gobwas/httphead v0.0.0-20200921212729-da3d93bc3c58/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo=github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=github.com/gobwas/ws v1.0.4/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM=@@ -261,15 +267,20 @@ github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg=github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY=github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw=github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI=github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ=+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=github.com/hashicorp/go-retryablehttp v0.7.1 h1:sUiuQAnLlbvmExtFQs72iFW/HXeUn8Z1aJLQ4LJJbTQ=github.com/hashicorp/go-retryablehttp v0.7.1/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY=github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=+github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=@@ -279,6 +290,8 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwHgithub.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A=github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo=github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4=+github.com/jlaffaye/ftp v0.1.0 h1:DLGExl5nBoSFoNshAUHwXAezXwXBvFdx7/qwhucWNSE=+github.com/jlaffaye/ftp v0.1.0/go.mod h1:hhq4G4crv+nW2qXtNYcuzLeOudG92Ps37HEKeg2e3lE=github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=@@ -337,6 +350,17 @@ github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYgithub.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno=github.com/nwaples/rardecode/v2 v2.0.0-beta.2 h1:e3mzJFJs4k83GXBEiTaQ5HgSc/kOK8q0rDaRO0MPaOk=github.com/nwaples/rardecode/v2 v2.0.0-beta.2/go.mod h1:yntwv/HfMc/Hbvtq9I19D1n58te3h6KsqCf3GxyfBGY=+github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A=+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=+github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=+github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=+github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=+github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=+github.com/onsi/gomega v1.23.0 h1:/oxKu9c2HVap+F3PfKort2Hw5DEU+HGlW8n+tguWsys=+github.com/onsi/gomega v1.23.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg=github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=github.com/paulbellamy/ratecounter v0.2.0 h1:2L/RhJq+HA8gBQImDXtLPrDXK5qAj6ozWVK/zFXVJGs=github.com/paulbellamy/ratecounter v0.2.0/go.mod h1:Hfx1hDpSGoqxkVVpBi/IlYD7kChlfo5C6hzIHwPqfFE=@@ -491,6 +515,7 @@ golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDgolang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=+golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=@@ -511,6 +536,7 @@ golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=+golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=@@ -527,8 +553,8 @@ golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxgolang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=golang.org/x/net v0.0.0-20220907135653-1e95f45603a7/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=-golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458 h1:MgJ6t2zo8v0tbmLCueaCbF1RM+TtB0rs3Lv8DGtOIpY=-golang.org/x/net v0.0.0-20221012135044-0b7e1fb9d458/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk=+golang.org/x/net v0.1.0 h1:hZ/3BUoy5aId7sCpA/Tc5lt8DkFgdVS2onTpJsZ/fl0=+golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco=golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=@@ -556,6 +582,7 @@ golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0 h1:cu5kTvlzcw1Q5S9f5ip1/cpigolang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=+golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=@@ -564,8 +591,11 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wgolang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=+golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=+golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=+golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=@@ -587,6 +617,7 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wgolang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=+golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=@@ -605,11 +636,11 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBcgolang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/sys v0.0.0-20220908150016-7ac13a9a928d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=-golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec h1:BkDtF2Ih9xZ7le9ndzTA7KJow28VbQW3odyk/8drmuI=-golang.org/x/sys v0.0.0-20220928140112-f11e5e49a4ec/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=+golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=+golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=-golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 h1:JGgROgKl9N8DuW20oFS5gxc+lE67/N3FcwmBPMe7ArY=golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=+golang.org/x/term v0.1.0 h1:g6Z6vPFA9dYBAF7DWcH6sCcOntplXsDKcliusYijMlw=golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=@@ -617,8 +648,9 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=-golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk=golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=+golang.org/x/text v0.4.0 h1:BrVqGRd7+k1DiOgtnFvAkoQEWQvBc25ouMJM6429SFg=+golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=@@ -670,6 +702,7 @@ golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fgolang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=golang.org/x/tools v0.0.0-20201211185031-d93e913c1a58/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=+golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0=@@ -788,8 +821,11 @@ gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI=+gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys=gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY=gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0=+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=gopkg.in/warnings.v0 v0.1.2 h1:wFXVbFY8DY5/xOe1ECiWdKCzZlxgshcYVNkBHstARME=gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRNI=gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=diff --git a/pkg/detectors/ftp/ftp.go b/pkg/detectors/ftp/ftp.gonew file mode 100644index 000000000..a7c34840b--- /dev/null+++ b/pkg/detectors/ftp/ftp.go@@ -0,0 +1,98 @@+package ftp++import (+ "context"+ "net/url"+ "regexp"+ "strings"+ "time"++ "github.com/jlaffaye/ftp"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time.+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(`\bftp://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"ftp://"}+}++// FromData will find and optionally verify URI secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ urlMatch := match[0]+ password := match[1]++ // Skip findings where the password only has "*" characters, this is a redacted password+ if strings.Trim(password, "*") == "" {+ continue+ }++ parsedURL, err := url.Parse(urlMatch)+ if err != nil {+ continue+ }+ if _, ok := parsedURL.User.Password(); !ok {+ continue+ }++ redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_FTP,+ Raw: []byte(urlMatch),+ Redacted: redact,+ }++ if verify {+ s.Verified = verifyFTP(ctx, parsedURL)+ }++ if !s.Verified {+ // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.+ if strings.HasPrefix(password, "$") {+ continue+ }+ }++ if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {+ continue+ }++ results = append(results, s)+ }++ return results, nil+}++func verifyFTP(ctx context.Context, u *url.URL) bool {+ host := u.Host+ if !strings.Contains(host, ":") {+ host = host + ":21"+ }++ c, err := ftp.Dial(host, ftp.DialWithTimeout(5*time.Second))+ if err != nil {+ return false+ }++ password, _ := u.User.Password()+ err = c.Login(u.User.Username(), password)++ return err == nil+}diff --git a/pkg/detectors/ftp/ftp_test.go b/pkg/detectors/ftp/ftp_test.gonew file mode 100644index 000000000..16a6d61bf--- /dev/null+++ b/pkg/detectors/ftp/ftp_test.go@@ -0,0 +1,109 @@+//go:build detectors+// +build detectors++package ftp++import (+ "context"+ "testing"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestURI_FromChunk(t *testing.T) {+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "bad scheme",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("file://user:[email protected]:123/wh/at/ever"),+ verify: true,+ },+ wantErr: false,+ },+ {+ name: "verified FTP",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ // https://dlptest.com/ftp-test/+ data: []byte("ftp://dlpuser:[email protected]"),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_FTP,+ Verified: true,+ Redacted: "ftp://dlpuser:*************************@ftp.dlptest.com",+ },+ },+ wantErr: false,+ },+ {+ name: "unverified FTP",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ // https://dlptest.com/ftp-test/+ data: []byte("ftp://dlpuser:[email protected]"),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_FTP,+ Verified: false,+ Redacted: "ftp://dlpuser:*******@ftp.dlptest.com",+ },+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ // if os.Getenv("FORCE_PASS_DIFF") == "true" {+ // return+ // }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/mongodb/mongodb.go b/pkg/detectors/mongodb/mongodb.goindex 357f8130c..2bd64e009 100644--- a/pkg/detectors/mongodb/mongodb.go+++ b/pkg/detectors/mongodb/mongodb.go@@ -21,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil)var (// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.- keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://(?:[^:]+:(?:[^@]+)?@)?(?:[^/]+|/.+.sock?,?)+(?:/([^/."*<>:|?]*))?\?(?:(.+=?=\S+)&?)+)\b`)+ keyPat = regexp.MustCompile(`\b(mongodb(\+srv)?://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+)\b`)// TODO: Add support for sharded cluster, replica set and Atlas Deployment.)diff --git a/pkg/detectors/npmtokenv2/npmtokenv2_test.go b/pkg/detectors/npmtokenv2/npmtokenv2_test.goindex 4714e13b7..2fd623b81 100644--- a/pkg/detectors/npmtokenv2/npmtokenv2_test.go+++ b/pkg/detectors/npmtokenv2/npmtokenv2_test.go@@ -1,7 +1,7 @@//go:build detectors// +build detectors-package npmtoken_new+package npmtokenv2import ("context"@@ -48,7 +48,7 @@ func TestNpmToken_New_FromChunk(t *testing.T) {},want: []detectors.Result{{- DetectorType: detectorspb.DetectorType_NpmToken_New,+ DetectorType: detectorspb.DetectorType_NpmToken,Verified: true,},},@@ -64,7 +64,7 @@ func TestNpmToken_New_FromChunk(t *testing.T) {},want: []detectors.Result{{- DetectorType: detectorspb.DetectorType_NpmToken_New,+ DetectorType: detectorspb.DetectorType_NpmToken,Verified: false,},},diff --git a/pkg/detectors/redis/redis.go b/pkg/detectors/redis/redis.gonew file mode 100644index 000000000..f68ab3b0f--- /dev/null+++ b/pkg/detectors/redis/redis.go@@ -0,0 +1,96 @@+package redis++import (+ "context"+ "net/url"+ "regexp"+ "strings"++ "github.com/go-redis/redis"++ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++type Scanner struct{}++// Ensure the Scanner satisfies the interface at compile time.+var _ detectors.Detector = (*Scanner)(nil)++var (+ keyPat = regexp.MustCompile(`\bredis://[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)+)++// Keywords are used for efficiently pre-filtering chunks.+// Use identifiers in the secret preferably, or the provider name.+func (s Scanner) Keywords() []string {+ return []string{"redis"}+}++// FromData will find and optionally verify URI secrets in a given set of bytes.+func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {+ dataStr := string(data)++ matches := keyPat.FindAllStringSubmatch(dataStr, -1)++ for _, match := range matches {+ urlMatch := match[0]+ password := match[1]++ // Skip findings where the password only has "*" characters, this is a redacted password+ if strings.Trim(password, "*") == "" {+ continue+ }++ parsedURL, err := url.Parse(urlMatch)+ if err != nil {+ continue+ }+ if _, ok := parsedURL.User.Password(); !ok {+ continue+ }++ redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))++ s := detectors.Result{+ DetectorType: detectorspb.DetectorType_Redis,+ Raw: []byte(urlMatch),+ Redacted: redact,+ }++ if verify {+ s.Verified = verifyRedis(ctx, parsedURL)+ }++ if !s.Verified {+ // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.+ if strings.HasPrefix(password, "$") {+ continue+ }+ }++ if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {+ continue+ }++ results = append(results, s)+ }++ return results, nil+}++func verifyRedis(ctx context.Context, u *url.URL) bool {+ opt, err := redis.ParseURL(u.String())+ if err != nil {+ return false+ }++ client := redis.NewClient(opt)++ status, err := client.Ping().Result()+ if err == nil && status == "PONG" {+ return true+ }++ return false+}diff --git a/pkg/detectors/redis/redis_test.go b/pkg/detectors/redis/redis_test.gonew file mode 100644index 000000000..6ad2084fb--- /dev/null+++ b/pkg/detectors/redis/redis_test.go@@ -0,0 +1,90 @@+//go:build detectors+// +build detectors++package redis++import (+ "context"+ "testing"++ "github.com/kylelemons/godebug/pretty"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors"+ "github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"+)++func TestURI_FromChunk(t *testing.T) {+ type args struct {+ ctx context.Context+ data []byte+ verify bool+ }+ tests := []struct {+ name string+ s Scanner+ args args+ want []detectors.Result+ wantErr bool+ }{+ {+ name: "bad scheme",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("file://user:[email protected]:123/wh/at/ever"),+ verify: true,+ },+ wantErr: false,+ },+ {+ name: "unverified Redis",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte("redis://user:[email protected]"),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_Redis,+ Verified: false,+ Redacted: "redis://user:*******@redis.com",+ },+ },+ wantErr: false,+ },+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ s := Scanner{}+ got, err := s.FromData(tt.args.ctx, tt.args.verify, tt.args.data)+ if (err != nil) != tt.wantErr {+ t.Errorf("URI.FromData() error = %v, wantErr %v", err, tt.wantErr)+ return+ }+ // if os.Getenv("FORCE_PASS_DIFF") == "true" {+ // return+ // }+ for i := range got {+ got[i].Raw = nil+ }+ if diff := pretty.Compare(got, tt.want); diff != "" {+ t.Errorf("URI.FromData() %s diff: (-got +want)\n%s", tt.name, diff)+ }+ })+ }+}++func BenchmarkFromData(benchmark *testing.B) {+ ctx := context.Background()+ s := Scanner{}+ for name, data := range detectors.MustGetBenchmarkData() {+ benchmark.Run(name, func(b *testing.B) {+ for n := 0; n < b.N; n++ {+ _, err := s.FromData(ctx, false, data)+ if err != nil {+ b.Fatal(err)+ }+ }+ })+ }+}diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.goindex 6a4278ef0..9112eaed7 100644--- a/pkg/detectors/uri/uri.go+++ b/pkg/detectors/uri/uri.go@@ -1,14 +1,12 @@package uriimport (- "bytes""context"- "encoding/json"- "io""net/http""net/url""regexp""strings"+ "time""github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors"@@ -23,7 +21,9 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- keyPat = regexp.MustCompile(`\b[a-zA-Z]{1,10}:?\/\/[-.%\w{}]{1,50}:([-.%\S]{3,50})@[-.%\w\/:]+\b`)+ keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)++ client = common.SaneHttpClient())type proxyRes struct {@@ -36,23 +36,8 @@ func (s Scanner) Keywords() []string {return []string{"http"}}-func allowlistedProtos(scheme string) bool {- allowlisted := []string{"http", "https", "mongodb", "redis", "ftp"}- for _, s := range allowlisted {- if s == scheme {- return true- }- }- return false-}-// FromData will find and optionally verify URI secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {- // Prevent SSRF "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9"- // TODO add as config option- // TODO extend to other http outbound calls in other areas of code- ssrfProtectorURL := "https://us-central1-ssrfproxy.cloudfunctions.net/ssrfproxy-6d96c399-74bb-4299-b49d-c1c870277eb9"-dataStr := string(data)matches := keyPat.FindAllStringSubmatch(dataStr, -1)@@ -71,11 +56,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resulturlMatch := match[0]password := match[1]- // Skip findings where the password starts with a `$` - it's almost certainly a variable.- if strings.HasPrefix(password, "$") {- continue- }-// Skip findings where the password only has "*" characters, this is a redacted passwordif strings.Trim(password, "*") == "" {continue@@ -88,9 +68,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif _, ok := parsedURL.User.Password(); !ok {continue}- if !allowlistedProtos(parsedURL.Scheme) {- continue- }redact := strings.TrimSpace(strings.Replace(urlMatch, password, strings.Repeat("*", len(password)), -1))@@ -101,29 +78,14 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- client := common.SaneHttpClient()- // whitelist protocols-- // Assume a 200 response is a valid credential- postValues := map[string]string{"protocol": parsedURL.Scheme, "credentialed_uri": urlMatch}- jsonValue, _ := json.Marshal(postValues)- req, err := http.NewRequestWithContext(ctx, "POST", ssrfProtectorURL, bytes.NewBuffer(jsonValue))- if err != nil {+ s.Verified = verifyURL(ctx, parsedURL)+ }++ if !s.Verified {+ // Skip unverified findings where the password starts with a `$` - it's almost certainly a variable.+ if strings.HasPrefix(password, "$") {continue}- req.Header.Add("Content-Type", "application/json")- res, err := client.Do(req)- if err == nil {- result := proxyRes{}- body, err := io.ReadAll(res.Body)- res.Body.Close()- if len(body) != 0 && err == nil {- err = json.Unmarshal(body, &result)- if err == nil && result.Verified {- s.Verified = true- }- }- }}if !s.Verified && detectors.IsKnownFalsePositive(string(s.Raw), detectors.DefaultFalsePositives, false) {@@ -135,3 +97,53 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn results, nil}++func verifyURL(ctx context.Context, u *url.URL) bool {+ // defuse most SSRF payloads+ u.Path = strings.TrimSuffix(u.Path, "/")+ u.RawQuery = ""+ u.Fragment = ""++ credentialedURL := u.String()++ u.User = nil+ nonCredentialedURL := u.String()++ req, err := http.NewRequest("GET", credentialedURL, nil)+ if err != nil {+ return false+ }+ req = req.WithContext(ctx)+ credentialedRes, err := client.Do(req)+ if err != nil {+ return false+ }+ credentialedRes.Body.Close()++ // If the credentialed URL returns a non 2XX code, we can assume it's a false positive.+ if credentialedRes.StatusCode < 200 || credentialedRes.StatusCode > 299 {+ return false+ }++ time.Sleep(time.Millisecond * 10)++ req, err = http.NewRequest("GET", nonCredentialedURL, nil)+ if err != nil {+ return false+ }+ req = req.WithContext(ctx)+ nonCredentialedRes, err := client.Do(req)+ if err != nil {+ return false+ }+ nonCredentialedRes.Body.Close()++ // If the non-credentialed URL returns a non 400-428 code and basic auth header, we can assume it's verified now.+ if nonCredentialedRes.StatusCode >= 400 && nonCredentialedRes.StatusCode < 429 {+ if nonCredentialedRes.Header.Get("WWW-Authenticate") != "" {+ return true+ }+ }++ return false+}diff --git a/pkg/detectors/uri/uri_test.go b/pkg/detectors/uri/uri_test.goindex c924a60d5..012b5d846 100644--- a/pkg/detectors/uri/uri_test.go+++ b/pkg/detectors/uri/uri_test.go@@ -7,23 +7,13 @@ import ("context""fmt""testing"- "time""github.com/kylelemons/godebug/pretty"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")func TestURI_FromChunk(t *testing.T) {- ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)- defer cancel()- testSecrets, err := common.GetSecret(ctx, "trufflehog-testing", "detectors2")- if err != nil {- t.Fatalf("could not get test secrets from GCP: %s", err)- }- secret := testSecrets.MustGetField("URI_SECRET")- secretInactive := testSecrets.MustGetField("URI_INACTIVE")type args struct {ctx context.Contextdata []byte@@ -36,20 +26,19 @@ func TestURI_FromChunk(t *testing.T) {want []detectors.ResultwantErr bool}{-{name: "found, unverified, wrong username",s: Scanner{},args: args{ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a uri secret %s within", secretInactive)),+ data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://user:[email protected]/httpgallery/authentication/authenticatedimage/default.aspx")),verify: true,},want: []detectors.Result{{DetectorType: detectorspb.DetectorType_URI,Verified: false,- Redacted: "https://user:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",+ Redacted: "https://user:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",},},wantErr: false,@@ -59,20 +48,37 @@ func TestURI_FromChunk(t *testing.T) {s: Scanner{},args: args{ctx: context.Background(),- data: []byte(fmt.Sprintf("You can find a uri secret %s within", secret)),+ data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://httpwatch:[email protected]/httpgallery/authentication/authenticatedimage/default.aspx")),+ verify: true,+ },+ want: []detectors.Result{+ {+ DetectorType: detectorspb.DetectorType_URI,+ Verified: true,+ Redacted: "https://httpwatch:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",+ },+ },+ wantErr: false,+ },+ {+ name: "found, verified, defused",+ s: Scanner{},+ args: args{+ ctx: context.Background(),+ data: []byte(fmt.Sprintf("You can find a uri secret %s within", "https://httpwatch:[email protected]/httpgallery/authentication/authenticatedimage/default.aspx?foo=bar")),verify: true,},want: []detectors.Result{{DetectorType: detectorspb.DetectorType_URI,Verified: true,- Redacted: "https://httpwatch:**********@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",+ Redacted: "https://httpwatch:****@www.httpwatch.com/httpgallery/authentication/authenticatedimage/default.aspx",},},wantErr: false,},{- name: "bad proto",+ name: "bad scheme",s: Scanner{},args: args{ctx: context.Background(),diff --git a/pkg/engine/defaults.go b/pkg/engine/defaults.goindex b5b8d10c0..1ea37ba4c 100644--- a/pkg/engine/defaults.go+++ b/pkg/engine/defaults.go@@ -255,6 +255,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors/freshbooks""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/freshdesk""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/front"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/ftp""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fulcrum""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fullstory""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/fusebill"@@ -510,6 +511,7 @@ import ("github.com/trufflesecurity/trufflehog/v3/pkg/detectors/reallysimplesystems""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rebrandly""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rechargepayments"+ "github.com/trufflesecurity/trufflehog/v3/pkg/detectors/redis""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/refiner""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/rentman""github.com/trufflesecurity/trufflehog/v3/pkg/detectors/repairshopr"@@ -1484,5 +1486,7 @@ func DefaultDetectors() []detectors.Detector {npmtoken.Scanner{},npmtokenv2.Scanner{},sqlserver.Scanner{},+ redis.Scanner{},+ ftp.Scanner{},}}diff --git a/pkg/pb/detectorspb/detectors.pb.go b/pkg/pb/detectorspb/detectors.pb.goindex 108890eb9..724bb8dd7 100644--- a/pkg/pb/detectorspb/detectors.pb.go+++ b/pkg/pb/detectorspb/detectors.pb.go@@ -967,6 +967,8 @@ const (DetectorType_NGC DetectorType = 896DetectorType_DigitalOceanV2 DetectorType = 897DetectorType_SQLServer DetectorType = 898+ DetectorType_FTP DetectorType = 899+ DetectorType_Redis DetectorType = 900)// Enum value maps for DetectorType.@@ -1867,6 +1869,8 @@ var (896: "NGC",897: "DigitalOceanV2",898: "SQLServer",+ 899: "FTP",+ 900: "Redis",}DetectorType_value = map[string]int32{"Alibaba": 0,@@ -2764,6 +2768,8 @@ var ("NGC": 896,"DigitalOceanV2": 897,"SQLServer": 898,+ "FTP": 899,+ "Redis": 900,})@@ -3128,7 +3134,7 @@ var file_detectors_proto_rawDesc = []byte{0x0a, 0x0b, 0x44, 0x65, 0x63, 0x6f, 0x64, 0x65, 0x72, 0x54, 0x79, 0x70, 0x65, 0x12, 0x0b, 0x0a,0x07, 0x55, 0x4e, 0x4b, 0x4e, 0x4f, 0x57, 0x4e, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, 0x50, 0x4c,0x41, 0x49, 0x4e, 0x10, 0x01, 0x12, 0x0a, 0x0a, 0x06, 0x42, 0x41, 0x53, 0x45, 0x36, 0x34, 0x10,- 0x02, 0x2a, 0xc1, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,+ 0x02, 0x2a, 0xd7, 0x70, 0x0a, 0x0c, 0x44, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x54, 0x79,0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x41, 0x6c, 0x69, 0x62, 0x61, 0x62, 0x61, 0x10, 0x00, 0x12,0x08, 0x0a, 0x04, 0x41, 0x4d, 0x51, 0x50, 0x10, 0x01, 0x12, 0x07, 0x0a, 0x03, 0x41, 0x57, 0x53,0x10, 0x02, 0x12, 0x09, 0x0a, 0x05, 0x41, 0x7a, 0x75, 0x72, 0x65, 0x10, 0x03, 0x12, 0x0a, 0x0a,@@ -4028,11 +4034,13 @@ var file_detectors_proto_rawDesc = []byte{0x6f, 0x44, 0x42, 0x10, 0xff, 0x06, 0x12, 0x08, 0x0a, 0x03, 0x4e, 0x47, 0x43, 0x10, 0x80, 0x07,0x12, 0x13, 0x0a, 0x0e, 0x44, 0x69, 0x67, 0x69, 0x74, 0x61, 0x6c, 0x4f, 0x63, 0x65, 0x61, 0x6e,0x56, 0x32, 0x10, 0x81, 0x07, 0x12, 0x0e, 0x0a, 0x09, 0x53, 0x51, 0x4c, 0x53, 0x65, 0x72, 0x76,- 0x65, 0x72, 0x10, 0x82, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e,- 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x73, 0x65, 0x63, 0x75, 0x72,- 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c, 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76,- 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64, 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f,- 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33,+ 0x65, 0x72, 0x10, 0x82, 0x07, 0x12, 0x08, 0x0a, 0x03, 0x46, 0x54, 0x50, 0x10, 0x83, 0x07, 0x12,+ 0x0a, 0x0a, 0x05, 0x52, 0x65, 0x64, 0x69, 0x73, 0x10, 0x84, 0x07, 0x42, 0x3d, 0x5a, 0x3b, 0x67,+ 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,+ 0x65, 0x73, 0x65, 0x63, 0x75, 0x72, 0x69, 0x74, 0x79, 0x2f, 0x74, 0x72, 0x75, 0x66, 0x66, 0x6c,+ 0x65, 0x68, 0x6f, 0x67, 0x2f, 0x76, 0x33, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x70, 0x62, 0x2f, 0x64,+ 0x65, 0x74, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x73, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74,+ 0x6f, 0x33,}var (diff --git a/proto/detectors.proto b/proto/detectors.protoindex e7e809165..289ea950a 100644--- a/proto/detectors.proto+++ b/proto/detectors.proto@@ -906,6 +906,8 @@ enum DetectorType {NGC = 896;DigitalOceanV2 = 897;SQLServer = 898;+ FTP = 899;+ Redis = 900;}message Result {