Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Improve domain / url handling in detectors (#3221)
Dustin Deckerfe5624c70923
diff --git a/pkg/detectors/aha/aha.go b/pkg/detectors/aha/aha.goindex 6788794aa..07d99671e 100644--- a/pkg/detectors/aha/aha.go+++ b/pkg/detectors/aha/aha.go@@ -8,7 +8,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -22,7 +21,7 @@ var (// Ensure the Scanner satisfies the interface at compile time._ detectors.Detector = (*Scanner)(nil)- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aha"}) + `\b([0-9a-f]{64})\b`)diff --git a/pkg/detectors/apiflash/apiflash.go b/pkg/detectors/apiflash/apiflash.goindex 50dc97ccb..d335267c5 100644--- a/pkg/detectors/apiflash/apiflash.go+++ b/pkg/detectors/apiflash/apiflash.go@@ -3,11 +3,11 @@ package apiflashimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apiflash"}) + `\b([a-z0-9]{32})\b`)diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.goindex afe9dd123..98841f984 100644--- a/pkg/detectors/artifactory/artifactory.go+++ b/pkg/detectors/artifactory/artifactory.go@@ -8,7 +8,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -22,7 +21,7 @@ var (// Ensure the Scanner satisfies the interface at compile time._ detectors.Detector = (*Scanner)(nil)- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`)diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.goindex 82b78697f..5d9a513ce 100644--- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go+++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go@@ -8,7 +8,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -22,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil)var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithLocalAddresses// long jwt token but note this is default 8640000 seconds = 24 hours but could be set to maximum 2592000 seconds = 720 hours = 30 days// at https://manage.auth0.com/dashboard/us/dev-63memjo3/apis/management/explorerdiff --git a/pkg/detectors/auth0oauth/auth0oauth.go b/pkg/detectors/auth0oauth/auth0oauth.goindex c35e14e4e..93bf956b7 100644--- a/pkg/detectors/auth0oauth/auth0oauth.go+++ b/pkg/detectors/auth0oauth/auth0oauth.go@@ -2,13 +2,13 @@ package auth0oauthimport ("context"- regexp "github.com/wasilibs/go-re2""io""net/http""net/url""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithLocalAddressesclientIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"auth0"}) + `\b([a-zA-Z0-9_-]{32,60})\b`)clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)@@ -102,7 +102,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif !strings.Contains(body, "access_denied") {s1.Verified = true- }+ }}}diff --git a/pkg/detectors/azurebatch/azurebatch.go b/pkg/detectors/azurebatch/azurebatch.goindex da5fd9333..4bad3e0eb 100644--- a/pkg/detectors/azurebatch/azurebatch.go+++ b/pkg/detectors/azurebatch/azurebatch.go@@ -12,7 +12,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -27,7 +26,7 @@ var _ detectors.Detector = (*Scanner)(nil)var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.urlPat = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`)secretPat = regexp.MustCompile(`[A-Za-z0-9+/=]{88}`)diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey.go b/pkg/detectors/azurefunctionkey/azurefunctionkey.goindex 7af938ea2..3aca16851 100644--- a/pkg/detectors/azurefunctionkey/azurefunctionkey.go+++ b/pkg/detectors/azurefunctionkey/azurefunctionkey.go@@ -8,7 +8,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -22,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([a-zA-Z0-9_-]{20,56})\b={0,2}`)azureUrlPat = regexp.MustCompile(`\bhttps:\/\/([a-zA-Z0-9-]{2,30})\.azurewebsites\.net\/api\/([a-zA-Z0-9-]{2,30})\b`)diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.goindex 5e7b9497e..5d1fed88e 100644--- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go+++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go@@ -8,7 +8,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -22,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z]{52})\b`)urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `https:\/\/([0-9a-z]{5,40})\.search\.windows\.net\/indexes\/([0-9a-z]{5,40})\b`)diff --git a/pkg/detectors/azurestorage/azurestorage.go b/pkg/detectors/azurestorage/azurestorage.goindex 6aa0791ed..84afdd98b 100644--- a/pkg/detectors/azurestorage/azurestorage.go+++ b/pkg/detectors/azurestorage/azurestorage.go@@ -25,7 +25,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = http.DefaultClient+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresseskeyPat = regexp.MustCompile(`DefaultEndpointsProtocol=https;AccountName=(?P<account_name>[^;]+);AccountKey=(?P<account_key>[^;]+);EndpointSuffix=core\.windows\.net`))diff --git a/pkg/detectors/caspio/caspio.go b/pkg/detectors/caspio/caspio.goindex b022e1123..195dc7237 100644--- a/pkg/detectors/caspio/caspio.go+++ b/pkg/detectors/caspio/caspio.go@@ -3,11 +3,11 @@ package caspioimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"caspio"}) + `\b([a-z0-9]{50})\b`)diff --git a/pkg/detectors/databrickstoken/databrickstoken.go b/pkg/detectors/databrickstoken/databrickstoken.goindex 866fcbb99..43bfd7016 100644--- a/pkg/detectors/databrickstoken/databrickstoken.go+++ b/pkg/detectors/databrickstoken/databrickstoken.go@@ -3,11 +3,11 @@ package databrickstokenimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.domain = regexp.MustCompile(`\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(cloud\.databricks\.com|gcp\.databricks\.com|azurewebsites\.net))\b`)diff --git a/pkg/detectors/deputy/deputy.go b/pkg/detectors/deputy/deputy.goindex ed0438fa5..dc1635151 100644--- a/pkg/detectors/deputy/deputy.go+++ b/pkg/detectors/deputy/deputy.go@@ -3,11 +3,11 @@ package deputyimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deputy"}) + `\b([0-9a-z]{32})\b`)diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex ee4f1d86b..b3e975ff8 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -241,3 +241,13 @@ func RedactURL(u url.URL) string {u.User = url.UserPassword(u.User.Username(), "********")return strings.TrimSpace(strings.Replace(u.String(), "%2A", "*", -1))}++func ParseURLAndStripPathAndParams(u string) (*url.URL, error) {+ parsedURL, err := url.Parse(u)+ if err != nil {+ return nil, err+ }+ parsedURL.Path = ""+ parsedURL.RawQuery = ""+ return parsedURL, nil+}diff --git a/pkg/detectors/discordwebhook/discordwebhook.go b/pkg/detectors/discordwebhook/discordwebhook.goindex e05a0fba2..c01e5677b 100644--- a/pkg/detectors/discordwebhook/discordwebhook.go+++ b/pkg/detectors/discordwebhook/discordwebhook.go@@ -7,7 +7,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -18,7 +17,7 @@ type Scanner struct{}var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(`(https:\/\/discord\.com\/api\/webhooks\/[0-9]{18}\/[0-9a-zA-Z-]{68})`)diff --git a/pkg/detectors/fibery/fibery.go b/pkg/detectors/fibery/fibery.goindex dbccd03c1..477c0987a 100644--- a/pkg/detectors/fibery/fibery.go+++ b/pkg/detectors/fibery/fibery.go@@ -3,12 +3,12 @@ package fiberyimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings""time"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"fibery"}) + `\b([0-9a-f]{8}.[0-9a-f]{35})\b`)diff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.goindex 75e65ce4d..9702ba87d 100644--- a/pkg/detectors/freshbooks/freshbooks.go+++ b/pkg/detectors/freshbooks/freshbooks.go@@ -3,12 +3,12 @@ package freshbooksimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""io""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,10 +21,10 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()-+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b([0-9a-z]{64})\b`)+ // TODO: this domain pattern is too restrictiveuriPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b(https://www.[0-9A-Za-z_-]{1,}.com)\b`))diff --git a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.goindex 6df3aaf89..78f5a4105 100644--- a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go+++ b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go@@ -3,11 +3,11 @@ package grafanaserviceaccountimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(`\b(glsa_[0-9a-zA-Z_]{41})\b`)domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.grafana\.net)\b`)diff --git a/pkg/detectors/http.go b/pkg/detectors/http.gonew file mode 100644index 000000000..a1780c1ce--- /dev/null+++ b/pkg/detectors/http.go@@ -0,0 +1,154 @@+package detectors++import (+ "context"+ "errors"+ "net"+ "net/http"+ "time"+)++var DetectorHttpClientWithNoLocalAddresses *http.Client+var DetectorHttpClientWithLocalAddresses *http.Client++const DefaultResponseTimeout = 5 * time.Second+const DefaultUserAgent = "TruffleHog"++func init() {+ DetectorHttpClientWithLocalAddresses = NewDetectorHttpClient(+ WithTransport(NewDetectorTransport(DefaultUserAgent, nil)),+ WithTimeout(DefaultResponseTimeout),+ WithNoFollowRedirects(),+ )+ DetectorHttpClientWithNoLocalAddresses = NewDetectorHttpClient(+ WithTransport(NewDetectorTransport(DefaultUserAgent, nil)),+ WithTimeout(DefaultResponseTimeout),+ WithNoFollowRedirects(),+ WithNoLocalIP(),+ )+}++// ClientOption defines a function type that modifies an http.Client.+type ClientOption func(*http.Client)++// WithNoFollowRedirects allows disabling automatic following of redirects.+func WithNoFollowRedirects() ClientOption {+ return func(c *http.Client) {+ c.CheckRedirect = func(req *http.Request, via []*http.Request) error {+ return http.ErrUseLastResponse+ }+ }+}++type detectorTransport struct {+ T http.RoundTripper+ userAgent string+}++func (t *detectorTransport) RoundTrip(req *http.Request) (*http.Response, error) {+ req.Header.Add("User-Agent", t.userAgent)+ return t.T.RoundTrip(req)+}++var defaultDialer = &net.Dialer{+ Timeout: 2 * time.Second,+ KeepAlive: 5 * time.Second,+}++func NewDetectorTransport(userAgent string, T http.RoundTripper) http.RoundTripper {+ if T == nil {+ T = &http.Transport{+ Proxy: http.ProxyFromEnvironment,+ DialContext: defaultDialer.DialContext,+ MaxIdleConns: 100,+ MaxIdleConnsPerHost: 5,+ IdleConnTimeout: 90 * time.Second,+ TLSHandshakeTimeout: 3 * time.Second,+ ExpectContinueTimeout: 1 * time.Second,+ }+ }+ return &detectorTransport{T: T, userAgent: userAgent}+}++func isLocalIP(ip net.IP) bool {+ if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() {+ return true+ }++ return false+}++var ErrNoLocalIP = errors.New("dialing local IP addresses is not allowed")++func WithNoLocalIP() ClientOption {+ return func(c *http.Client) {+ if c.Transport == nil {+ c.Transport = &http.Transport{}+ }++ // Type assertion to get the underlying *http.Transport+ transport, ok := c.Transport.(*http.Transport)+ if !ok {+ // If c.Transport is not *http.Transport, check if it is wrapped in a detectorTransport+ dt, ok := c.Transport.(*detectorTransport)+ if !ok {+ panic("unsupported transport type")+ }+ transport, ok = dt.T.(*http.Transport)+ if !ok {+ panic("underlying transport is not *http.Transport")+ }+ }++ // If the original DialContext is nil, set it to the default dialer+ if transport.DialContext == nil {+ transport.DialContext = defaultDialer.DialContext+ }+ originalDialContext := transport.DialContext+ transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {+ host, port, err := net.SplitHostPort(addr)+ if err != nil {+ return nil, err+ }++ ips, err := net.LookupIP(host)+ if err != nil {+ return nil, err+ }++ for _, ip := range ips {+ if isLocalIP(ip) {+ return nil, ErrNoLocalIP+ }+ }++ return originalDialContext(ctx, network, net.JoinHostPort(host, port))+ }+ }+}++// WithTransport sets a custom transport for the http.Client.+func WithTransport(transport http.RoundTripper) ClientOption {+ return func(c *http.Client) {+ c.Transport = transport+ }+}++// WithTimeout sets a timeout for the http.Client.+func WithTimeout(timeout time.Duration) ClientOption {+ return func(c *http.Client) {+ c.Timeout = timeout+ }+}++func NewDetectorHttpClient(opts ...ClientOption) *http.Client {+ httpClient := &http.Client{+ Transport: NewDetectorTransport(DefaultUserAgent, nil),+ Timeout: DefaultResponseTimeout,+ }++ for _, opt := range opts {+ opt(httpClient)+ }+ return httpClient+}diff --git a/pkg/detectors/http_test.go b/pkg/detectors/http_test.gonew file mode 100644index 000000000..bbe6b9dfe--- /dev/null+++ b/pkg/detectors/http_test.go@@ -0,0 +1,94 @@+package detectors++import (+ "context"+ "net"+ "net/http"+ "testing"++ "github.com/stretchr/testify/assert"+)++func TestWithNoLocalIP(t *testing.T) {+ t.Run("Prevents dialing local IP", func(t *testing.T) {+ client := &http.Client{}+ WithNoLocalIP()(client)++ transport, ok := client.Transport.(*http.Transport)+ assert.True(t, ok, "Expected transport to be *http.Transport")++ _, err := transport.DialContext(context.Background(), "tcp", "127.0.0.1:8080")+ assert.Error(t, err)+ assert.ErrorIs(t, err, ErrNoLocalIP)+ })++ t.Run("Allows dialing non-local host", func(t *testing.T) {+ client := &http.Client{}+ WithNoLocalIP()(client)++ transport, ok := client.Transport.(*http.Transport)+ assert.True(t, ok, "Expected transport to be *http.Transport")++ conn, err := transport.DialContext(context.Background(), "tcp", "google.com:80")+ assert.NoError(t, err)+ assert.NotNil(t, conn)+ conn.Close()+ })++ t.Run("Allows dialing non-local IP", func(t *testing.T) {+ client := &http.Client{}+ WithNoLocalIP()(client)++ transport, ok := client.Transport.(*http.Transport)+ assert.True(t, ok, "Expected transport to be *http.Transport")++ conn, err := transport.DialContext(context.Background(), "tcp", "1.1.1.1:80")+ assert.NoError(t, err)+ assert.NotNil(t, conn)+ conn.Close()+ })++ t.Run("Handles invalid address", func(t *testing.T) {+ client := &http.Client{}+ WithNoLocalIP()(client)++ transport, ok := client.Transport.(*http.Transport)+ assert.True(t, ok, "Expected transport to be *http.Transport")++ _, err := transport.DialContext(context.Background(), "tcp", "invalid-address")+ assert.Error(t, err)+ })++ t.Run("Handles non-existent hostname", func(t *testing.T) {+ client := &http.Client{}+ WithNoLocalIP()(client)++ transport, ok := client.Transport.(*http.Transport)+ assert.True(t, ok, "Expected transport to be *http.Transport")++ _, err := transport.DialContext(context.Background(), "tcp", "non-existent-host.local:80")+ assert.Error(t, err)+ })+}++func TestIsLocalIP(t *testing.T) {+ testCases := []struct {+ name string+ ip net.IP+ expected bool+ }{+ {"Loopback IPv4", net.ParseIP("127.0.0.1"), true},+ {"Loopback IPv6", net.ParseIP("::1"), true},+ {"Private IPv4", net.ParseIP("192.168.1.1"), true},+ {"Private IPv6", net.ParseIP("fd00::1"), true},+ {"Public IPv4", net.ParseIP("8.8.8.8"), false},+ {"Public IPv6", net.ParseIP("2001:4860:4860::8888"), false},+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ result := isLocalIP(tc.ip)+ assert.Equal(t, tc.expected, result)+ })+ }+}diff --git a/pkg/detectors/invoiceocean/invoiceocean.go b/pkg/detectors/invoiceocean/invoiceocean.goindex f95a4ec4c..2d8add465 100644--- a/pkg/detectors/invoiceocean/invoiceocean.go+++ b/pkg/detectors/invoiceocean/invoiceocean.go@@ -3,11 +3,11 @@ package invoiceoceanimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"invoiceocean"}) + `\b([0-9A-Za-z]{20})\b`)diff --git a/pkg/detectors/jiratoken/v1/jiratoken.go b/pkg/detectors/jiratoken/v1/jiratoken.goindex ae23754e5..988ac6700 100644--- a/pkg/detectors/jiratoken/v1/jiratoken.go+++ b/pkg/detectors/jiratoken/v1/jiratoken.go@@ -9,7 +9,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -26,7 +25,7 @@ var _ detectors.Versioner = (*Scanner)(nil)func (Scanner) Version() int { return 1 }var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2.go b/pkg/detectors/jiratoken/v2/jiratoken_v2.goindex 3501bde9b..cac0a5d3a 100644--- a/pkg/detectors/jiratoken/v2/jiratoken_v2.go+++ b/pkg/detectors/jiratoken/v2/jiratoken_v2.go@@ -9,7 +9,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -26,7 +25,7 @@ var _ detectors.Versioner = (*Scanner)(nil)func (Scanner) Version() int { return 2 }var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithLocalAddresses// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/// Tokens created after Jan 18 2023 use a variable lengthdiff --git a/pkg/detectors/kanban/kanban.go b/pkg/detectors/kanban/kanban.goindex d3fb775ea..8cb4dfe50 100644--- a/pkg/detectors/kanban/kanban.go+++ b/pkg/detectors/kanban/kanban.go@@ -3,11 +3,11 @@ package kanbanimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanban"}) + `\b([0-9A-Z]{12})\b`)diff --git a/pkg/detectors/kanbantool/kanbantool.go b/pkg/detectors/kanbantool/kanbantool.goindex a6bb8e23e..3fad48c98 100644--- a/pkg/detectors/kanbantool/kanbantool.go+++ b/pkg/detectors/kanbantool/kanbantool.go@@ -3,11 +3,11 @@ package kanbantoolimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positiveskeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanbantool"}) + `\b([0-9A-Z]{12})\b`)diff --git a/pkg/detectors/liveagent/liveagent.go b/pkg/detectors/liveagent/liveagent.goindex a4ed7f394..603e9f1ca 100644--- a/pkg/detectors/liveagent/liveagent.go+++ b/pkg/detectors/liveagent/liveagent.go@@ -3,11 +3,11 @@ package liveagentimport ("context""encoding/json"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.domainPat = regexp.MustCompile(`\b(https?://[A-Za-z0-9-]+\.ladesk\.com)\b`)diff --git a/pkg/detectors/loggly/loggly.go b/pkg/detectors/loggly/loggly.goindex 4de7e41d6..b89f77715 100644--- a/pkg/detectors/loggly/loggly.go+++ b/pkg/detectors/loggly/loggly.go@@ -3,11 +3,11 @@ package logglyimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.loggly\.com)\b`)keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loggly"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)diff --git a/pkg/detectors/metabase/metabase.go b/pkg/detectors/metabase/metabase.goindex 66fd9dc9a..b2d8815db 100644--- a/pkg/detectors/metabase/metabase.go+++ b/pkg/detectors/metabase/metabase.go@@ -3,12 +3,12 @@ package metabaseimport ("context""encoding/json"- regexp "github.com/wasilibs/go-re2""io""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"metabase"}) + `\b([a-zA-Z0-9-]{36})\b`)@@ -54,6 +54,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}resURLMatch := strings.TrimSpace(urlMatch[1])+ u, err := detectors.ParseURLAndStripPathAndParams(resURLMatch)+ if err != nil {+ // if the URL is invalid just move onto the next one+ continue+ }+s1 := detectors.Result{DetectorType: detectorspb.DetectorType_Metabase,Raw: []byte(resMatch),@@ -61,7 +67,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}if verify {- req, err := http.NewRequestWithContext(ctx, http.MethodGet, resURLMatch+"/api/user/current", nil)+ u.Path = "/api/user/current"+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)if err != nil {continue}diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.goindex 446da7a81..31816704e 100644--- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go+++ b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go@@ -6,11 +6,9 @@ import ("io""net/http""strings"- "time"regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -24,7 +22,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClientTimeOut(5 * time.Second)+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)diff --git a/pkg/detectors/mite/mite.go b/pkg/detectors/mite/mite.goindex 9d98c76a6..a068cb678 100644--- a/pkg/detectors/mite/mite.go+++ b/pkg/detectors/mite/mite.go@@ -3,11 +3,11 @@ package miteimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mite"}) + `\b([0-9a-z]{16})\b`)diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.goindex d3e8f2a2b..9642d2b3a 100644--- a/pkg/detectors/okta/okta.go+++ b/pkg/detectors/okta/okta.go@@ -9,7 +9,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -64,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreq.Header.Set("Content-Type", "application/json")req.Header.Set("Authorization", fmt.Sprintf("SSWS %s", token))- resp, err := common.SaneHttpClient().Do(req)+ resp, err := detectors.DetectorHttpClientWithNoLocalAddresses.Do(req)if err != nil {continue}diff --git a/pkg/detectors/openvpn/openvpn.go b/pkg/detectors/openvpn/openvpn.goindex 96fea975d..8fc24973b 100644--- a/pkg/detectors/openvpn/openvpn.go+++ b/pkg/detectors/openvpn/openvpn.go@@ -9,7 +9,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -23,7 +22,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddressesclientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openvpn"}) + `\b([A-Za-z0-9-]{3,40}\.[A-Za-z0-9-]{3,40})\b`)clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)diff --git a/pkg/detectors/planviewleankit/planviewleankit.go b/pkg/detectors/planviewleankit/planviewleankit.goindex f4547dc19..8c565352b 100644--- a/pkg/detectors/planviewleankit/planviewleankit.go+++ b/pkg/detectors/planviewleankit/planviewleankit.go@@ -3,11 +3,11 @@ package planviewleankitimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"planviewleankit", "planview"}) + `\b([0-9a-f]{128})\b`)diff --git a/pkg/detectors/portainer/portainer.go b/pkg/detectors/portainer/portainer.goindex aee0be138..56af42e7e 100644--- a/pkg/detectors/portainer/portainer.go+++ b/pkg/detectors/portainer/portainer.go@@ -3,11 +3,11 @@ package portainerimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{50,310}\.[0-9A-Z-a-z\-_]{43})\b`)@@ -49,6 +49,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultfor _, endpointMatch := range endpointMatches {resEndpointMatch := strings.TrimSpace(endpointMatch[1])+ u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch)+ if err != nil {+ // if the URL is invalid just move onto the next one+ continue+ }+ u.Path = "/api/endpoints"+s1 := detectors.Result{DetectorType: detectorspb.DetectorType_Portainer,Raw: []byte(resMatch),@@ -60,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif client == nil {client = defaultClient}- req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch+"/api/endpoints", nil)+ req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)if err != nil {continue}@@ -81,8 +88,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result}}--if len(endpointMatches) > 0 {results = append(results, s1)}diff --git a/pkg/detectors/portainertoken/portainertoken.go b/pkg/detectors/portainertoken/portainertoken.goindex 54ad63c1d..53b88df1d 100644--- a/pkg/detectors/portainertoken/portainertoken.go+++ b/pkg/detectors/portainertoken/portainertoken.go@@ -3,11 +3,11 @@ package portainertokenimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainertoken"}) + `\b(ptr_[A-Za-z0-9\/_\-+=]{20,60})`)endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)@@ -49,6 +49,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultfor _, endpointMatch := range endpointMatches {resEndpointMatch := strings.TrimSpace(endpointMatch[1])+ u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch)+ if err != nil {+ // if the URL is invalid just move onto the next one+ continue+ }+ u.Path = "/api/stacks"+s1 := detectors.Result{DetectorType: detectorspb.DetectorType_PortainerToken,Raw: []byte(resMatch),@@ -60,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultif client == nil {client = defaultClient}- req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch+"/api/stacks", nil)+ req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)if err != nil {continue}diff --git a/pkg/detectors/repairshopr/repairshopr.go b/pkg/detectors/repairshopr/repairshopr.goindex 47976fa9d..58ee10fb6 100644--- a/pkg/detectors/repairshopr/repairshopr.go+++ b/pkg/detectors/repairshopr/repairshopr.go@@ -3,11 +3,11 @@ package repairshoprimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"repairshopr"}) + `\b([a-zA-Z0-9-]{51})\b`)diff --git a/pkg/detectors/ringcentral/ringcentral.go b/pkg/detectors/ringcentral/ringcentral.goindex a71075512..11dff8a5f 100644--- a/pkg/detectors/ringcentral/ringcentral.go+++ b/pkg/detectors/ringcentral/ringcentral.go@@ -3,10 +3,11 @@ package ringcentralimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -24,6 +25,7 @@ var (// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ringcentral"}) + `\b([0-9A-Za-z_-]{22})\b`)+ // TODO: this domain pattern is too restrictiveuriPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ringcentral"}) + `\b(https://www.[0-9A-Za-z_-]{1,}.com)\b`))diff --git a/pkg/detectors/salesforce/salesforce.go b/pkg/detectors/salesforce/salesforce.goindex e220ad5ca..f74e24d92 100644--- a/pkg/detectors/salesforce/salesforce.go+++ b/pkg/detectors/salesforce/salesforce.go@@ -3,10 +3,11 @@ package salesforceimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"@@ -25,7 +26,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.accessTokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9]{13}![a-zA-Z0-9_.]{96}\b`)instancePat = regexp.MustCompile(`\bhttps://[0-9a-zA-Z-\.]{1,100}\.my\.salesforce\.com\b`)diff --git a/pkg/detectors/salesmate/salesmate.go b/pkg/detectors/salesmate/salesmate.goindex 141502d72..27112a1c0 100644--- a/pkg/detectors/salesmate/salesmate.go+++ b/pkg/detectors/salesmate/salesmate.go@@ -3,11 +3,11 @@ package salesmateimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -20,7 +20,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positiveskeyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"salesmate"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.goindex 00ac455bb..13b11ecf9 100644--- a/pkg/detectors/shopify/shopify.go+++ b/pkg/detectors/shopify/shopify.go@@ -8,7 +8,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -22,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil)var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(`\b(shppa_|shpat_)([0-9A-Fa-f]{32})\b`)diff --git a/pkg/detectors/signalwire/signalwire.go b/pkg/detectors/signalwire/signalwire.goindex 743cb5587..570aeb3b7 100644--- a/pkg/detectors/signalwire/signalwire.go+++ b/pkg/detectors/signalwire/signalwire.go@@ -4,11 +4,11 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"signalwire"}) + `\b([0-9A-Za-z]{50})\b`)diff --git a/pkg/detectors/signalwire/signalwire_test.go b/pkg/detectors/signalwire/signalwire_test.goindex 50fc14142..a4c06f265 100644--- a/pkg/detectors/signalwire/signalwire_test.go+++ b/pkg/detectors/signalwire/signalwire_test.go@@ -23,7 +23,7 @@ func TestSignalwire_FromChunk(t *testing.T) {if err != nil {t.Fatalf("could not get test secrets from GCP: %s", err)}- id := testSecrets.MustGetField("SGINALWIRE_ID")+ id := testSecrets.MustGetField("SIGNALWIRE_ID")secret := testSecrets.MustGetField("SIGNALWIRE_TOKEN")url := testSecrets.MustGetField("SIGNALWIRE_URL")inactiveSecret := testSecrets.MustGetField("SIGNALWIRE_INACTIVE")diff --git a/pkg/detectors/slackwebhook/slackwebhook.go b/pkg/detectors/slackwebhook/slackwebhook.goindex d8042114b..508501fe6 100644--- a/pkg/detectors/slackwebhook/slackwebhook.go+++ b/pkg/detectors/slackwebhook/slackwebhook.go@@ -8,7 +8,6 @@ import ("net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"regexp "github.com/wasilibs/go-re2"@@ -23,7 +22,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPats = map[string]*regexp.Regexp{"Slack Service Web Hook": regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`),diff --git a/pkg/detectors/sugester/sugester.go b/pkg/detectors/sugester/sugester.goindex e22be1d9e..c604e61dd 100644--- a/pkg/detectors/sugester/sugester.go+++ b/pkg/detectors/sugester/sugester.go@@ -2,11 +2,11 @@ package sugesterimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -19,7 +19,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sugester"}) + `\b([a-zA-Z0-9]{32})\b`)diff --git a/pkg/detectors/thinkific/thinkific.go b/pkg/detectors/thinkific/thinkific.goindex d0cb6631d..fdfbcb638 100644--- a/pkg/detectors/thinkific/thinkific.go+++ b/pkg/detectors/thinkific/thinkific.go@@ -3,12 +3,12 @@ package thinkificimport ("context""fmt"- regexp "github.com/wasilibs/go-re2""io""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"thinkific"}) + `\b([0-9a-f]{32})\b`)diff --git a/pkg/detectors/tineswebhook/tineswebhook.go b/pkg/detectors/tineswebhook/tineswebhook.goindex 7126e2d7c..09654eb79 100644--- a/pkg/detectors/tineswebhook/tineswebhook.go+++ b/pkg/detectors/tineswebhook/tineswebhook.go@@ -7,7 +7,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -18,7 +17,7 @@ type Scanner struct{}var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(`(https://[\w-]+\.tines\.com/webhook/[a-z0-9]{32}/[a-z0-9]{32})`)@@ -56,9 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultres, err := client.Do(req)if err == nil {defer res.Body.Close()- if err != nil {- continue- }if res.StatusCode >= 200 && res.StatusCode < 300 {s1.Verified = true}diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.goindex cd8af3004..e383ac9b9 100644--- a/pkg/detectors/uri/uri.go+++ b/pkg/detectors/uri/uri.go@@ -9,7 +9,6 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -26,7 +25,8 @@ var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)var (keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)- defaultClient = common.SaneHttpClient()+ // TODO: make local addr opt-out+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses)// Keywords are used for efficiently pre-filtering chunks.diff --git a/pkg/detectors/zapierwebhook/zapierwebhook.go b/pkg/detectors/zapierwebhook/zapierwebhook.goindex 9f1d45ffe..faaad087b 100644--- a/pkg/detectors/zapierwebhook/zapierwebhook.go+++ b/pkg/detectors/zapierwebhook/zapierwebhook.go@@ -2,11 +2,11 @@ package zapierwebhookimport ("context"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -17,7 +17,7 @@ type Scanner struct{}var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(`(https:\/\/hooks\.zapier\.com\/hooks\/catch\/[A-Za-z0-9\/]{16})`)diff --git a/pkg/detectors/zendeskapi/zendeskapi.go b/pkg/detectors/zendeskapi/zendeskapi.goindex 913723a5d..3d40cca4c 100644--- a/pkg/detectors/zendeskapi/zendeskapi.go+++ b/pkg/detectors/zendeskapi/zendeskapi.go@@ -4,11 +4,11 @@ import ("context"b64 "encoding/base64""fmt"- regexp "github.com/wasilibs/go-re2""net/http""strings"- "github.com/trufflesecurity/trufflehog/v3/pkg/common"+ regexp "github.com/wasilibs/go-re2"+"github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")@@ -21,7 +21,7 @@ type Scanner struct{var _ detectors.Detector = (*Scanner)(nil)var (- client = common.SaneHttpClient()+ client = detectors.DetectorHttpClientWithNoLocalAddressestoken = regexp.MustCompile(detectors.PrefixRegex([]string{"zendesk"}) + `([A-Za-z0-9_-]{40})`)email = regexp.MustCompile(`\b([a-zA-Z-0-9-]{5,16}\@[a-zA-Z-0-9]{4,16}\.[a-zA-Z-0-9]{3,6})\b`)diff --git a/pkg/detectors/zulipchat/zulipchat.go b/pkg/detectors/zulipchat/zulipchat.goindex eb5e3d956..661d03f20 100644--- a/pkg/detectors/zulipchat/zulipchat.go+++ b/pkg/detectors/zulipchat/zulipchat.go@@ -23,7 +23,7 @@ type Scanner struct {var _ detectors.Detector = (*Scanner)(nil)var (- defaultClient = common.SaneHttpClient()+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.keyPat = regexp.MustCompile(common.BuildRegex(common.AlphaNumPattern, "", 32))