Skip to content

Find, verify, and analyze leaked credentials

Improve domain / url handling in detectors (#3221)

Dustin Deckerfe5624c70923
diff --git a/pkg/detectors/aha/aha.go b/pkg/detectors/aha/aha.go
index 6788794aa..07d99671e 100644
--- a/pkg/detectors/aha/aha.go
+++ b/pkg/detectors/aha/aha.go
@@ -8,7 +8,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -22,7 +21,7 @@ var (
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"aha"}) + `\b([0-9a-f]{64})\b`)
diff --git a/pkg/detectors/apiflash/apiflash.go b/pkg/detectors/apiflash/apiflash.go
index 50dc97ccb..d335267c5 100644
--- a/pkg/detectors/apiflash/apiflash.go
+++ b/pkg/detectors/apiflash/apiflash.go
@@ -3,11 +3,11 @@ package apiflash
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"apiflash"}) + `\b([a-z0-9]{32})\b`)
diff --git a/pkg/detectors/artifactory/artifactory.go b/pkg/detectors/artifactory/artifactory.go
index afe9dd123..98841f984 100644
--- a/pkg/detectors/artifactory/artifactory.go
+++ b/pkg/detectors/artifactory/artifactory.go
@@ -8,7 +8,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -22,7 +21,7 @@ var (
// Ensure the Scanner satisfies the interface at compile time.
_ detectors.Detector = (*Scanner)(nil)
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b([a-zA-Z0-9]{73}|\b[a-zA-Z0-9]{64})`)
diff --git a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go
index 82b78697f..5d9a513ce 100644
--- a/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go
+++ b/pkg/detectors/auth0managementapitoken/auth0managementapitoken.go
@@ -8,7 +8,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -22,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.MaxSecretSizeProvider = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithLocalAddresses
// long jwt token but note this is default 8640000 seconds = 24 hours but could be set to maximum 2592000 seconds = 720 hours = 30 days
// at https://manage.auth0.com/dashboard/us/dev-63memjo3/apis/management/explorer
diff --git a/pkg/detectors/auth0oauth/auth0oauth.go b/pkg/detectors/auth0oauth/auth0oauth.go
index c35e14e4e..93bf956b7 100644
--- a/pkg/detectors/auth0oauth/auth0oauth.go
+++ b/pkg/detectors/auth0oauth/auth0oauth.go
@@ -2,13 +2,13 @@ package auth0oauth
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"net/url"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithLocalAddresses
clientIdPat = regexp.MustCompile(detectors.PrefixRegex([]string{"auth0"}) + `\b([a-zA-Z0-9_-]{32,60})\b`)
clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)
@@ -102,7 +102,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if !strings.Contains(body, "access_denied") {
s1.Verified = true
- }
+ }
}
}
diff --git a/pkg/detectors/azurebatch/azurebatch.go b/pkg/detectors/azurebatch/azurebatch.go
index da5fd9333..4bad3e0eb 100644
--- a/pkg/detectors/azurebatch/azurebatch.go
+++ b/pkg/detectors/azurebatch/azurebatch.go
@@ -12,7 +12,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -27,7 +26,7 @@ var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
urlPat = regexp.MustCompile(`https://(.{1,50})\.(.{1,50})\.batch\.azure\.com`)
secretPat = regexp.MustCompile(`[A-Za-z0-9+/=]{88}`)
diff --git a/pkg/detectors/azurefunctionkey/azurefunctionkey.go b/pkg/detectors/azurefunctionkey/azurefunctionkey.go
index 7af938ea2..3aca16851 100644
--- a/pkg/detectors/azurefunctionkey/azurefunctionkey.go
+++ b/pkg/detectors/azurefunctionkey/azurefunctionkey.go
@@ -8,7 +8,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -22,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([a-zA-Z0-9_-]{20,56})\b={0,2}`)
azureUrlPat = regexp.MustCompile(`\bhttps:\/\/([a-zA-Z0-9-]{2,30})\.azurewebsites\.net\/api\/([a-zA-Z0-9-]{2,30})\b`)
diff --git a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go
index 5e7b9497e..5d1fed88e 100644
--- a/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go
+++ b/pkg/detectors/azuresearchquerykey/azuresearchquerykey.go
@@ -8,7 +8,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -22,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `\b([0-9a-zA-Z]{52})\b`)
urlPat = regexp.MustCompile(detectors.PrefixRegex([]string{"azure"}) + `https:\/\/([0-9a-z]{5,40})\.search\.windows\.net\/indexes\/([0-9a-z]{5,40})\b`)
diff --git a/pkg/detectors/azurestorage/azurestorage.go b/pkg/detectors/azurestorage/azurestorage.go
index 6aa0791ed..84afdd98b 100644
--- a/pkg/detectors/azurestorage/azurestorage.go
+++ b/pkg/detectors/azurestorage/azurestorage.go
@@ -25,7 +25,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = http.DefaultClient
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
keyPat = regexp.MustCompile(`DefaultEndpointsProtocol=https;AccountName=(?P<account_name>[^;]+);AccountKey=(?P<account_key>[^;]+);EndpointSuffix=core\.windows\.net`)
)
diff --git a/pkg/detectors/caspio/caspio.go b/pkg/detectors/caspio/caspio.go
index b022e1123..195dc7237 100644
--- a/pkg/detectors/caspio/caspio.go
+++ b/pkg/detectors/caspio/caspio.go
@@ -3,11 +3,11 @@ package caspio
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"caspio"}) + `\b([a-z0-9]{50})\b`)
diff --git a/pkg/detectors/databrickstoken/databrickstoken.go b/pkg/detectors/databrickstoken/databrickstoken.go
index 866fcbb99..43bfd7016 100644
--- a/pkg/detectors/databrickstoken/databrickstoken.go
+++ b/pkg/detectors/databrickstoken/databrickstoken.go
@@ -3,11 +3,11 @@ package databrickstoken
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
domain = regexp.MustCompile(`\b([a-z0-9-]+(?:\.[a-z0-9-]+)*\.(cloud\.databricks\.com|gcp\.databricks\.com|azurewebsites\.net))\b`)
diff --git a/pkg/detectors/deputy/deputy.go b/pkg/detectors/deputy/deputy.go
index ed0438fa5..dc1635151 100644
--- a/pkg/detectors/deputy/deputy.go
+++ b/pkg/detectors/deputy/deputy.go
@@ -3,11 +3,11 @@ package deputy
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"deputy"}) + `\b([0-9a-z]{32})\b`)
diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go
index ee4f1d86b..b3e975ff8 100644
--- a/pkg/detectors/detectors.go
+++ b/pkg/detectors/detectors.go
@@ -241,3 +241,13 @@ func RedactURL(u url.URL) string {
u.User = url.UserPassword(u.User.Username(), "********")
return strings.TrimSpace(strings.Replace(u.String(), "%2A", "*", -1))
}
+
+func ParseURLAndStripPathAndParams(u string) (*url.URL, error) {
+ parsedURL, err := url.Parse(u)
+ if err != nil {
+ return nil, err
+ }
+ parsedURL.Path = ""
+ parsedURL.RawQuery = ""
+ return parsedURL, nil
+}
diff --git a/pkg/detectors/discordwebhook/discordwebhook.go b/pkg/detectors/discordwebhook/discordwebhook.go
index e05a0fba2..c01e5677b 100644
--- a/pkg/detectors/discordwebhook/discordwebhook.go
+++ b/pkg/detectors/discordwebhook/discordwebhook.go
@@ -7,7 +7,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -18,7 +17,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`(https:\/\/discord\.com\/api\/webhooks\/[0-9]{18}\/[0-9a-zA-Z-]{68})`)
diff --git a/pkg/detectors/fibery/fibery.go b/pkg/detectors/fibery/fibery.go
index dbccd03c1..477c0987a 100644
--- a/pkg/detectors/fibery/fibery.go
+++ b/pkg/detectors/fibery/fibery.go
@@ -3,12 +3,12 @@ package fibery
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
"time"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"fibery"}) + `\b([0-9a-f]{8}.[0-9a-f]{35})\b`)
diff --git a/pkg/detectors/freshbooks/freshbooks.go b/pkg/detectors/freshbooks/freshbooks.go
index 75e65ce4d..9702ba87d 100644
--- a/pkg/detectors/freshbooks/freshbooks.go
+++ b/pkg/detectors/freshbooks/freshbooks.go
@@ -3,12 +3,12 @@ package freshbooks
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,10 +21,10 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
-
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b([0-9a-z]{64})\b`)
+ // TODO: this domain pattern is too restrictive
uriPat = regexp.MustCompile(detectors.PrefixRegex([]string{"freshbooks"}) + `\b(https://www.[0-9A-Za-z_-]{1,}.com)\b`)
)
diff --git a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go
index 6df3aaf89..78f5a4105 100644
--- a/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go
+++ b/pkg/detectors/grafanaserviceaccount/grafanaserviceaccount.go
@@ -3,11 +3,11 @@ package grafanaserviceaccount
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(glsa_[0-9a-zA-Z_]{41})\b`)
domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.grafana\.net)\b`)
diff --git a/pkg/detectors/http.go b/pkg/detectors/http.go
new file mode 100644
index 000000000..a1780c1ce
--- /dev/null
+++ b/pkg/detectors/http.go
@@ -0,0 +1,154 @@
+package detectors
+
+import (
+ "context"
+ "errors"
+ "net"
+ "net/http"
+ "time"
+)
+
+var DetectorHttpClientWithNoLocalAddresses *http.Client
+var DetectorHttpClientWithLocalAddresses *http.Client
+
+const DefaultResponseTimeout = 5 * time.Second
+const DefaultUserAgent = "TruffleHog"
+
+func init() {
+ DetectorHttpClientWithLocalAddresses = NewDetectorHttpClient(
+ WithTransport(NewDetectorTransport(DefaultUserAgent, nil)),
+ WithTimeout(DefaultResponseTimeout),
+ WithNoFollowRedirects(),
+ )
+ DetectorHttpClientWithNoLocalAddresses = NewDetectorHttpClient(
+ WithTransport(NewDetectorTransport(DefaultUserAgent, nil)),
+ WithTimeout(DefaultResponseTimeout),
+ WithNoFollowRedirects(),
+ WithNoLocalIP(),
+ )
+}
+
+// ClientOption defines a function type that modifies an http.Client.
+type ClientOption func(*http.Client)
+
+// WithNoFollowRedirects allows disabling automatic following of redirects.
+func WithNoFollowRedirects() ClientOption {
+ return func(c *http.Client) {
+ c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+ return http.ErrUseLastResponse
+ }
+ }
+}
+
+type detectorTransport struct {
+ T http.RoundTripper
+ userAgent string
+}
+
+func (t *detectorTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+ req.Header.Add("User-Agent", t.userAgent)
+ return t.T.RoundTrip(req)
+}
+
+var defaultDialer = &net.Dialer{
+ Timeout: 2 * time.Second,
+ KeepAlive: 5 * time.Second,
+}
+
+func NewDetectorTransport(userAgent string, T http.RoundTripper) http.RoundTripper {
+ if T == nil {
+ T = &http.Transport{
+ Proxy: http.ProxyFromEnvironment,
+ DialContext: defaultDialer.DialContext,
+ MaxIdleConns: 100,
+ MaxIdleConnsPerHost: 5,
+ IdleConnTimeout: 90 * time.Second,
+ TLSHandshakeTimeout: 3 * time.Second,
+ ExpectContinueTimeout: 1 * time.Second,
+ }
+ }
+ return &detectorTransport{T: T, userAgent: userAgent}
+}
+
+func isLocalIP(ip net.IP) bool {
+ if ip.IsLoopback() || ip.IsLinkLocalUnicast() || ip.IsLinkLocalMulticast() || ip.IsPrivate() {
+ return true
+ }
+
+ return false
+}
+
+var ErrNoLocalIP = errors.New("dialing local IP addresses is not allowed")
+
+func WithNoLocalIP() ClientOption {
+ return func(c *http.Client) {
+ if c.Transport == nil {
+ c.Transport = &http.Transport{}
+ }
+
+ // Type assertion to get the underlying *http.Transport
+ transport, ok := c.Transport.(*http.Transport)
+ if !ok {
+ // If c.Transport is not *http.Transport, check if it is wrapped in a detectorTransport
+ dt, ok := c.Transport.(*detectorTransport)
+ if !ok {
+ panic("unsupported transport type")
+ }
+ transport, ok = dt.T.(*http.Transport)
+ if !ok {
+ panic("underlying transport is not *http.Transport")
+ }
+ }
+
+ // If the original DialContext is nil, set it to the default dialer
+ if transport.DialContext == nil {
+ transport.DialContext = defaultDialer.DialContext
+ }
+ originalDialContext := transport.DialContext
+ transport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) {
+ host, port, err := net.SplitHostPort(addr)
+ if err != nil {
+ return nil, err
+ }
+
+ ips, err := net.LookupIP(host)
+ if err != nil {
+ return nil, err
+ }
+
+ for _, ip := range ips {
+ if isLocalIP(ip) {
+ return nil, ErrNoLocalIP
+ }
+ }
+
+ return originalDialContext(ctx, network, net.JoinHostPort(host, port))
+ }
+ }
+}
+
+// WithTransport sets a custom transport for the http.Client.
+func WithTransport(transport http.RoundTripper) ClientOption {
+ return func(c *http.Client) {
+ c.Transport = transport
+ }
+}
+
+// WithTimeout sets a timeout for the http.Client.
+func WithTimeout(timeout time.Duration) ClientOption {
+ return func(c *http.Client) {
+ c.Timeout = timeout
+ }
+}
+
+func NewDetectorHttpClient(opts ...ClientOption) *http.Client {
+ httpClient := &http.Client{
+ Transport: NewDetectorTransport(DefaultUserAgent, nil),
+ Timeout: DefaultResponseTimeout,
+ }
+
+ for _, opt := range opts {
+ opt(httpClient)
+ }
+ return httpClient
+}
diff --git a/pkg/detectors/http_test.go b/pkg/detectors/http_test.go
new file mode 100644
index 000000000..bbe6b9dfe
--- /dev/null
+++ b/pkg/detectors/http_test.go
@@ -0,0 +1,94 @@
+package detectors
+
+import (
+ "context"
+ "net"
+ "net/http"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestWithNoLocalIP(t *testing.T) {
+ t.Run("Prevents dialing local IP", func(t *testing.T) {
+ client := &http.Client{}
+ WithNoLocalIP()(client)
+
+ transport, ok := client.Transport.(*http.Transport)
+ assert.True(t, ok, "Expected transport to be *http.Transport")
+
+ _, err := transport.DialContext(context.Background(), "tcp", "127.0.0.1:8080")
+ assert.Error(t, err)
+ assert.ErrorIs(t, err, ErrNoLocalIP)
+ })
+
+ t.Run("Allows dialing non-local host", func(t *testing.T) {
+ client := &http.Client{}
+ WithNoLocalIP()(client)
+
+ transport, ok := client.Transport.(*http.Transport)
+ assert.True(t, ok, "Expected transport to be *http.Transport")
+
+ conn, err := transport.DialContext(context.Background(), "tcp", "google.com:80")
+ assert.NoError(t, err)
+ assert.NotNil(t, conn)
+ conn.Close()
+ })
+
+ t.Run("Allows dialing non-local IP", func(t *testing.T) {
+ client := &http.Client{}
+ WithNoLocalIP()(client)
+
+ transport, ok := client.Transport.(*http.Transport)
+ assert.True(t, ok, "Expected transport to be *http.Transport")
+
+ conn, err := transport.DialContext(context.Background(), "tcp", "1.1.1.1:80")
+ assert.NoError(t, err)
+ assert.NotNil(t, conn)
+ conn.Close()
+ })
+
+ t.Run("Handles invalid address", func(t *testing.T) {
+ client := &http.Client{}
+ WithNoLocalIP()(client)
+
+ transport, ok := client.Transport.(*http.Transport)
+ assert.True(t, ok, "Expected transport to be *http.Transport")
+
+ _, err := transport.DialContext(context.Background(), "tcp", "invalid-address")
+ assert.Error(t, err)
+ })
+
+ t.Run("Handles non-existent hostname", func(t *testing.T) {
+ client := &http.Client{}
+ WithNoLocalIP()(client)
+
+ transport, ok := client.Transport.(*http.Transport)
+ assert.True(t, ok, "Expected transport to be *http.Transport")
+
+ _, err := transport.DialContext(context.Background(), "tcp", "non-existent-host.local:80")
+ assert.Error(t, err)
+ })
+}
+
+func TestIsLocalIP(t *testing.T) {
+ testCases := []struct {
+ name string
+ ip net.IP
+ expected bool
+ }{
+ {"Loopback IPv4", net.ParseIP("127.0.0.1"), true},
+ {"Loopback IPv6", net.ParseIP("::1"), true},
+ {"Private IPv4", net.ParseIP("192.168.1.1"), true},
+ {"Private IPv6", net.ParseIP("fd00::1"), true},
+ {"Public IPv4", net.ParseIP("8.8.8.8"), false},
+ {"Public IPv6", net.ParseIP("2001:4860:4860::8888"), false},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ result := isLocalIP(tc.ip)
+ assert.Equal(t, tc.expected, result)
+ })
+ }
+}
diff --git a/pkg/detectors/invoiceocean/invoiceocean.go b/pkg/detectors/invoiceocean/invoiceocean.go
index f95a4ec4c..2d8add465 100644
--- a/pkg/detectors/invoiceocean/invoiceocean.go
+++ b/pkg/detectors/invoiceocean/invoiceocean.go
@@ -3,11 +3,11 @@ package invoiceocean
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"invoiceocean"}) + `\b([0-9A-Za-z]{20})\b`)
diff --git a/pkg/detectors/jiratoken/v1/jiratoken.go b/pkg/detectors/jiratoken/v1/jiratoken.go
index ae23754e5..988ac6700 100644
--- a/pkg/detectors/jiratoken/v1/jiratoken.go
+++ b/pkg/detectors/jiratoken/v1/jiratoken.go
@@ -9,7 +9,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -26,7 +25,7 @@ var _ detectors.Versioner = (*Scanner)(nil)
func (Scanner) Version() int { return 1 }
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"jira"}) + `\b([a-zA-Z-0-9]{24})\b`)
diff --git a/pkg/detectors/jiratoken/v2/jiratoken_v2.go b/pkg/detectors/jiratoken/v2/jiratoken_v2.go
index 3501bde9b..cac0a5d3a 100644
--- a/pkg/detectors/jiratoken/v2/jiratoken_v2.go
+++ b/pkg/detectors/jiratoken/v2/jiratoken_v2.go
@@ -9,7 +9,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -26,7 +25,7 @@ var _ detectors.Versioner = (*Scanner)(nil)
func (Scanner) Version() int { return 2 }
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithLocalAddresses
// https://support.atlassian.com/atlassian-account/docs/manage-api-tokens-for-your-atlassian-account/
// Tokens created after Jan 18 2023 use a variable length
diff --git a/pkg/detectors/kanban/kanban.go b/pkg/detectors/kanban/kanban.go
index d3fb775ea..8cb4dfe50 100644
--- a/pkg/detectors/kanban/kanban.go
+++ b/pkg/detectors/kanban/kanban.go
@@ -3,11 +3,11 @@ package kanban
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanban"}) + `\b([0-9A-Z]{12})\b`)
diff --git a/pkg/detectors/kanbantool/kanbantool.go b/pkg/detectors/kanbantool/kanbantool.go
index a6bb8e23e..3fad48c98 100644
--- a/pkg/detectors/kanbantool/kanbantool.go
+++ b/pkg/detectors/kanbantool/kanbantool.go
@@ -3,11 +3,11 @@ package kanbantool
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"kanbantool"}) + `\b([0-9A-Z]{12})\b`)
diff --git a/pkg/detectors/liveagent/liveagent.go b/pkg/detectors/liveagent/liveagent.go
index a4ed7f394..603e9f1ca 100644
--- a/pkg/detectors/liveagent/liveagent.go
+++ b/pkg/detectors/liveagent/liveagent.go
@@ -3,11 +3,11 @@ package liveagent
import (
"context"
"encoding/json"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
domainPat = regexp.MustCompile(`\b(https?://[A-Za-z0-9-]+\.ladesk\.com)\b`)
diff --git a/pkg/detectors/loggly/loggly.go b/pkg/detectors/loggly/loggly.go
index 4de7e41d6..b89f77715 100644
--- a/pkg/detectors/loggly/loggly.go
+++ b/pkg/detectors/loggly/loggly.go
@@ -3,11 +3,11 @@ package loggly
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
domainPat = regexp.MustCompile(`\b([a-zA-Z0-9-]+\.loggly\.com)\b`)
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"loggly"}) + `\b([a-z0-9]{8}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{4}-[a-z0-9]{12})\b`)
diff --git a/pkg/detectors/metabase/metabase.go b/pkg/detectors/metabase/metabase.go
index 66fd9dc9a..b2d8815db 100644
--- a/pkg/detectors/metabase/metabase.go
+++ b/pkg/detectors/metabase/metabase.go
@@ -3,12 +3,12 @@ package metabase
import (
"context"
"encoding/json"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"metabase"}) + `\b([a-zA-Z0-9-]{36})\b`)
@@ -54,6 +54,12 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
resURLMatch := strings.TrimSpace(urlMatch[1])
+ u, err := detectors.ParseURLAndStripPathAndParams(resURLMatch)
+ if err != nil {
+ // if the URL is invalid just move onto the next one
+ continue
+ }
+
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Metabase,
Raw: []byte(resMatch),
@@ -61,7 +67,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
if verify {
- req, err := http.NewRequestWithContext(ctx, http.MethodGet, resURLMatch+"/api/user/current", nil)
+ u.Path = "/api/user/current"
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
if err != nil {
continue
}
diff --git a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go
index 446da7a81..31816704e 100644
--- a/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go
+++ b/pkg/detectors/microsoftteamswebhook/microsoftteamswebhook.go
@@ -6,11 +6,9 @@ import (
"io"
"net/http"
"strings"
- "time"
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -24,7 +22,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClientTimeOut(5 * time.Second)
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`(https:\/\/[a-zA-Z-0-9]+\.webhook\.office\.com\/webhookb2\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\@[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12}\/IncomingWebhook\/[a-zA-Z-0-9]{32}\/[a-zA-Z-0-9]{8}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{4}-[a-zA-Z-0-9]{12})`)
diff --git a/pkg/detectors/mite/mite.go b/pkg/detectors/mite/mite.go
index 9d98c76a6..a068cb678 100644
--- a/pkg/detectors/mite/mite.go
+++ b/pkg/detectors/mite/mite.go
@@ -3,11 +3,11 @@ package mite
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"mite"}) + `\b([0-9a-z]{16})\b`)
diff --git a/pkg/detectors/okta/okta.go b/pkg/detectors/okta/okta.go
index d3e8f2a2b..9642d2b3a 100644
--- a/pkg/detectors/okta/okta.go
+++ b/pkg/detectors/okta/okta.go
@@ -9,7 +9,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -64,7 +63,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("SSWS %s", token))
- resp, err := common.SaneHttpClient().Do(req)
+ resp, err := detectors.DetectorHttpClientWithNoLocalAddresses.Do(req)
if err != nil {
continue
}
diff --git a/pkg/detectors/openvpn/openvpn.go b/pkg/detectors/openvpn/openvpn.go
index 96fea975d..8fc24973b 100644
--- a/pkg/detectors/openvpn/openvpn.go
+++ b/pkg/detectors/openvpn/openvpn.go
@@ -9,7 +9,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -23,7 +22,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
clientIDPat = regexp.MustCompile(detectors.PrefixRegex([]string{"openvpn"}) + `\b([A-Za-z0-9-]{3,40}\.[A-Za-z0-9-]{3,40})\b`)
clientSecretPat = regexp.MustCompile(`\b([a-zA-Z0-9_-]{64,})\b`)
diff --git a/pkg/detectors/planviewleankit/planviewleankit.go b/pkg/detectors/planviewleankit/planviewleankit.go
index f4547dc19..8c565352b 100644
--- a/pkg/detectors/planviewleankit/planviewleankit.go
+++ b/pkg/detectors/planviewleankit/planviewleankit.go
@@ -3,11 +3,11 @@ package planviewleankit
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"planviewleankit", "planview"}) + `\b([0-9a-f]{128})\b`)
diff --git a/pkg/detectors/portainer/portainer.go b/pkg/detectors/portainer/portainer.go
index aee0be138..56af42e7e 100644
--- a/pkg/detectors/portainer/portainer.go
+++ b/pkg/detectors/portainer/portainer.go
@@ -3,11 +3,11 @@ package portainer
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)
tokenPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9\.[0-9A-Za-z]{50,310}\.[0-9A-Z-a-z\-_]{43})\b`)
@@ -49,6 +49,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
for _, endpointMatch := range endpointMatches {
resEndpointMatch := strings.TrimSpace(endpointMatch[1])
+ u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch)
+ if err != nil {
+ // if the URL is invalid just move onto the next one
+ continue
+ }
+ u.Path = "/api/endpoints"
+
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_Portainer,
Raw: []byte(resMatch),
@@ -60,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if client == nil {
client = defaultClient
}
- req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch+"/api/endpoints", nil)
+ req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {
continue
}
@@ -81,8 +88,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
}
}
-
-
if len(endpointMatches) > 0 {
results = append(results, s1)
}
diff --git a/pkg/detectors/portainertoken/portainertoken.go b/pkg/detectors/portainertoken/portainertoken.go
index 54ad63c1d..53b88df1d 100644
--- a/pkg/detectors/portainertoken/portainertoken.go
+++ b/pkg/detectors/portainertoken/portainertoken.go
@@ -3,11 +3,11 @@ package portainertoken
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainertoken"}) + `\b(ptr_[A-Za-z0-9\/_\-+=]{20,60})`)
endpointPat = regexp.MustCompile(detectors.PrefixRegex([]string{"portainer"}) + `\b(https?:\/\/\S+(:[0-9]{4,5})?)\b`)
@@ -49,6 +49,13 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
for _, endpointMatch := range endpointMatches {
resEndpointMatch := strings.TrimSpace(endpointMatch[1])
+ u, err := detectors.ParseURLAndStripPathAndParams(resEndpointMatch)
+ if err != nil {
+ // if the URL is invalid just move onto the next one
+ continue
+ }
+ u.Path = "/api/stacks"
+
s1 := detectors.Result{
DetectorType: detectorspb.DetectorType_PortainerToken,
Raw: []byte(resMatch),
@@ -60,7 +67,7 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
if client == nil {
client = defaultClient
}
- req, err := http.NewRequestWithContext(ctx, "GET", resEndpointMatch+"/api/stacks", nil)
+ req, err := http.NewRequestWithContext(ctx, "GET", u.String(), nil)
if err != nil {
continue
}
diff --git a/pkg/detectors/repairshopr/repairshopr.go b/pkg/detectors/repairshopr/repairshopr.go
index 47976fa9d..58ee10fb6 100644
--- a/pkg/detectors/repairshopr/repairshopr.go
+++ b/pkg/detectors/repairshopr/repairshopr.go
@@ -3,11 +3,11 @@ package repairshopr
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"repairshopr"}) + `\b([a-zA-Z0-9-]{51})\b`)
diff --git a/pkg/detectors/ringcentral/ringcentral.go b/pkg/detectors/ringcentral/ringcentral.go
index a71075512..11dff8a5f 100644
--- a/pkg/detectors/ringcentral/ringcentral.go
+++ b/pkg/detectors/ringcentral/ringcentral.go
@@ -3,10 +3,11 @@ package ringcentral
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -24,6 +25,7 @@ var (
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ringcentral"}) + `\b([0-9A-Za-z_-]{22})\b`)
+ // TODO: this domain pattern is too restrictive
uriPat = regexp.MustCompile(detectors.PrefixRegex([]string{"ringcentral"}) + `\b(https://www.[0-9A-Za-z_-]{1,}.com)\b`)
)
diff --git a/pkg/detectors/salesforce/salesforce.go b/pkg/detectors/salesforce/salesforce.go
index e220ad5ca..f74e24d92 100644
--- a/pkg/detectors/salesforce/salesforce.go
+++ b/pkg/detectors/salesforce/salesforce.go
@@ -3,10 +3,11 @@ package salesforce
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
@@ -25,7 +26,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
accessTokenPat = regexp.MustCompile(`\b00[a-zA-Z0-9]{13}![a-zA-Z0-9_.]{96}\b`)
instancePat = regexp.MustCompile(`\bhttps://[0-9a-zA-Z-\.]{1,100}\.my\.salesforce\.com\b`)
diff --git a/pkg/detectors/salesmate/salesmate.go b/pkg/detectors/salesmate/salesmate.go
index 141502d72..27112a1c0 100644
--- a/pkg/detectors/salesmate/salesmate.go
+++ b/pkg/detectors/salesmate/salesmate.go
@@ -3,11 +3,11 @@ package salesmate
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -20,7 +20,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"salesmate"}) + `\b([0-9Aa-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})\b`)
diff --git a/pkg/detectors/shopify/shopify.go b/pkg/detectors/shopify/shopify.go
index 00ac455bb..13b11ecf9 100644
--- a/pkg/detectors/shopify/shopify.go
+++ b/pkg/detectors/shopify/shopify.go
@@ -8,7 +8,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -22,7 +21,7 @@ var _ detectors.Detector = (*Scanner)(nil)
var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`\b(shppa_|shpat_)([0-9A-Fa-f]{32})\b`)
diff --git a/pkg/detectors/signalwire/signalwire.go b/pkg/detectors/signalwire/signalwire.go
index 743cb5587..570aeb3b7 100644
--- a/pkg/detectors/signalwire/signalwire.go
+++ b/pkg/detectors/signalwire/signalwire.go
@@ -4,11 +4,11 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"signalwire"}) + `\b([0-9A-Za-z]{50})\b`)
diff --git a/pkg/detectors/signalwire/signalwire_test.go b/pkg/detectors/signalwire/signalwire_test.go
index 50fc14142..a4c06f265 100644
--- a/pkg/detectors/signalwire/signalwire_test.go
+++ b/pkg/detectors/signalwire/signalwire_test.go
@@ -23,7 +23,7 @@ func TestSignalwire_FromChunk(t *testing.T) {
if err != nil {
t.Fatalf("could not get test secrets from GCP: %s", err)
}
- id := testSecrets.MustGetField("SGINALWIRE_ID")
+ id := testSecrets.MustGetField("SIGNALWIRE_ID")
secret := testSecrets.MustGetField("SIGNALWIRE_TOKEN")
url := testSecrets.MustGetField("SIGNALWIRE_URL")
inactiveSecret := testSecrets.MustGetField("SIGNALWIRE_INACTIVE")
diff --git a/pkg/detectors/slackwebhook/slackwebhook.go b/pkg/detectors/slackwebhook/slackwebhook.go
index d8042114b..508501fe6 100644
--- a/pkg/detectors/slackwebhook/slackwebhook.go
+++ b/pkg/detectors/slackwebhook/slackwebhook.go
@@ -8,7 +8,6 @@ import (
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
regexp "github.com/wasilibs/go-re2"
@@ -23,7 +22,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPats = map[string]*regexp.Regexp{
"Slack Service Web Hook": regexp.MustCompile(`(https://hooks\.slack\.com/services/T[A-Z0-9]+/B[A-Z0-9]+/[A-Za-z0-9]{23,25})`),
diff --git a/pkg/detectors/sugester/sugester.go b/pkg/detectors/sugester/sugester.go
index e22be1d9e..c604e61dd 100644
--- a/pkg/detectors/sugester/sugester.go
+++ b/pkg/detectors/sugester/sugester.go
@@ -2,11 +2,11 @@ package sugester
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -19,7 +19,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"sugester"}) + `\b([a-zA-Z0-9]{32})\b`)
diff --git a/pkg/detectors/thinkific/thinkific.go b/pkg/detectors/thinkific/thinkific.go
index d0cb6631d..fdfbcb638 100644
--- a/pkg/detectors/thinkific/thinkific.go
+++ b/pkg/detectors/thinkific/thinkific.go
@@ -3,12 +3,12 @@ package thinkific
import (
"context"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"io"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(detectors.PrefixRegex([]string{"thinkific"}) + `\b([0-9a-f]{32})\b`)
diff --git a/pkg/detectors/tineswebhook/tineswebhook.go b/pkg/detectors/tineswebhook/tineswebhook.go
index 7126e2d7c..09654eb79 100644
--- a/pkg/detectors/tineswebhook/tineswebhook.go
+++ b/pkg/detectors/tineswebhook/tineswebhook.go
@@ -7,7 +7,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -18,7 +17,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`(https://[\w-]+\.tines\.com/webhook/[a-z0-9]{32}/[a-z0-9]{32})`)
@@ -56,9 +55,6 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
res, err := client.Do(req)
if err == nil {
defer res.Body.Close()
- if err != nil {
- continue
- }
if res.StatusCode >= 200 && res.StatusCode < 300 {
s1.Verified = true
}
diff --git a/pkg/detectors/uri/uri.go b/pkg/detectors/uri/uri.go
index cd8af3004..e383ac9b9 100644
--- a/pkg/detectors/uri/uri.go
+++ b/pkg/detectors/uri/uri.go
@@ -9,7 +9,6 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -26,7 +25,8 @@ var _ detectors.CustomFalsePositiveChecker = (*Scanner)(nil)
var (
keyPat = regexp.MustCompile(`\b(?:https?:)?\/\/[\S]{3,50}:([\S]{3,50})@[-.%\w\/:]+\b`)
- defaultClient = common.SaneHttpClient()
+ // TODO: make local addr opt-out
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
)
// Keywords are used for efficiently pre-filtering chunks.
diff --git a/pkg/detectors/zapierwebhook/zapierwebhook.go b/pkg/detectors/zapierwebhook/zapierwebhook.go
index 9f1d45ffe..faaad087b 100644
--- a/pkg/detectors/zapierwebhook/zapierwebhook.go
+++ b/pkg/detectors/zapierwebhook/zapierwebhook.go
@@ -2,11 +2,11 @@ package zapierwebhook
import (
"context"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -17,7 +17,7 @@ type Scanner struct{}
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(`(https:\/\/hooks\.zapier\.com\/hooks\/catch\/[A-Za-z0-9\/]{16})`)
diff --git a/pkg/detectors/zendeskapi/zendeskapi.go b/pkg/detectors/zendeskapi/zendeskapi.go
index 913723a5d..3d40cca4c 100644
--- a/pkg/detectors/zendeskapi/zendeskapi.go
+++ b/pkg/detectors/zendeskapi/zendeskapi.go
@@ -4,11 +4,11 @@ import (
"context"
b64 "encoding/base64"
"fmt"
- regexp "github.com/wasilibs/go-re2"
"net/http"
"strings"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
+ regexp "github.com/wasilibs/go-re2"
+
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
@@ -21,7 +21,7 @@ type Scanner struct{
var _ detectors.Detector = (*Scanner)(nil)
var (
- client = common.SaneHttpClient()
+ client = detectors.DetectorHttpClientWithNoLocalAddresses
token = regexp.MustCompile(detectors.PrefixRegex([]string{"zendesk"}) + `([A-Za-z0-9_-]{40})`)
email = regexp.MustCompile(`\b([a-zA-Z-0-9-]{5,16}\@[a-zA-Z-0-9]{4,16}\.[a-zA-Z-0-9]{3,6})\b`)
diff --git a/pkg/detectors/zulipchat/zulipchat.go b/pkg/detectors/zulipchat/zulipchat.go
index eb5e3d956..661d03f20 100644
--- a/pkg/detectors/zulipchat/zulipchat.go
+++ b/pkg/detectors/zulipchat/zulipchat.go
@@ -23,7 +23,7 @@ type Scanner struct {
var _ detectors.Detector = (*Scanner)(nil)
var (
- defaultClient = common.SaneHttpClient()
+ defaultClient = detectors.DetectorHttpClientWithNoLocalAddresses
// Make sure that your group is surrounded in boundary characters such as below to reduce false positives.
keyPat = regexp.MustCompile(common.BuildRegex(common.AlphaNumPattern, "", 32))