Skip to content

Find, verify, and analyze leaked credentials

Rework JWT detector to better block local IPs; add HTTP instrumentation (#4607)

Brad Larsen728d71fbb3a9
diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.go
index 473e52714..d002db2af 100644
--- a/pkg/detectors/detectors.go
+++ b/pkg/detectors/detectors.go
@@ -18,15 +18,21 @@ import (
// Detector defines an interface for scanning for and verifying secrets.
type Detector interface {
// FromData will scan bytes for results and optionally verify them.
+ //
+ // FromData can be called concurrently from multiple goroutines.
+ // Any modification to the receiver or to global variables will need to to use some kind of synchronization.
FromData(ctx context.Context, verify bool, data []byte) ([]Result, error)
+
// Keywords are used for efficiently pre-filtering chunks using substring operations.
// Use unique identifiers that are part of the secret if you can, or the provider name.
//
// When multiple keywords are provided, they are is treated as a *union* of filtering terms.
// That is, if any of the keywords are found in a chunk, the chunk will be run through the detector.
Keywords() []string
+
// Type returns the DetectorType number from detectors.proto for the given detector.
Type() detectorspb.DetectorType
+
// Description returns a description for the result being detected
Description() string
}
@@ -93,7 +99,7 @@ type Result struct {
// DetectorName is the name of the Detector. Used for custom detectors.
DetectorName string
// Verified indicates whether the result was verified or not.
- Verified bool
+ Verified bool
// VerificationFromCache indicates whether this result's verification result came from the verification cache rather
// than an actual remote request.
VerificationFromCache bool
@@ -293,7 +299,7 @@ func MustGetBenchmarkData() map[string][]byte {
for key, size := range sizes {
// Generating a byte slice of a specific size with random data.
content := make([]byte, size)
- for i := range(size) {
+ for i := range size {
randomByte, err := rand.Int(rand.Reader, big.NewInt(256))
if err != nil {
panic(err)
diff --git a/pkg/detectors/http.go b/pkg/detectors/http.go
index ed7d21c5d..43ad5052c 100644
--- a/pkg/detectors/http.go
+++ b/pkg/detectors/http.go
@@ -5,9 +5,11 @@ import (
"errors"
"net"
"net/http"
+ "slices"
"sync"
"time"
+ "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
)
@@ -44,14 +46,12 @@ var overrideOnce sync.Once
// It is guaranteed to only run once, subsequent calls will have no effect.
// This should be called before any scans are started.
func OverrideDetectorTimeout(timeout time.Duration) {
- overrideOnce.Do(func() {
- DetectorHttpClientWithLocalAddresses.Timeout = timeout
- DetectorHttpClientWithNoLocalAddresses.Timeout = timeout
- })
+ overrideOnce.Do(func() {
+ DetectorHttpClientWithLocalAddresses.Timeout = timeout
+ DetectorHttpClientWithNoLocalAddresses.Timeout = timeout
+ })
}
-
-
// ClientOption defines a function type that modifies an http.Client.
type ClientOption func(*http.Client)
@@ -139,10 +139,8 @@ func WithNoLocalIP() ClientOption {
return nil, err
}
- for _, ip := range ips {
- if isLocalIP(ip) {
- return nil, ErrNoLocalIP
- }
+ if slices.ContainsFunc(ips, isLocalIP) {
+ return nil, ErrNoLocalIP
}
return originalDialContext(ctx, network, net.JoinHostPort(host, port))
@@ -165,13 +163,15 @@ func WithTimeout(timeout time.Duration) ClientOption {
}
func NewDetectorHttpClient(opts ...ClientOption) *http.Client {
- httpClient := &http.Client{
+ client := &http.Client{
Transport: NewDetectorTransport(nil),
Timeout: DefaultResponseTimeout,
}
for _, opt := range opts {
- opt(httpClient)
+ opt(client)
}
- return httpClient
+
+ client.Transport = common.NewInstrumentedTransport(client.Transport)
+ return client
}
diff --git a/pkg/detectors/jwt/jwt.go b/pkg/detectors/jwt/jwt.go
index f69d5b7ac..03d6e69d6 100644
--- a/pkg/detectors/jwt/jwt.go
+++ b/pkg/detectors/jwt/jwt.go
@@ -1,12 +1,10 @@
package jwt
import (
- "cmp"
"context"
"encoding/json"
"fmt"
"io"
- "net"
"net/http"
"net/url"
"strings"
@@ -17,14 +15,11 @@ import (
regexp "github.com/wasilibs/go-re2"
- "github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/detectors"
"github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb"
)
-type Scanner struct {
- client *http.Client
-}
+type Scanner struct{}
// Ensure the Scanner satisfies expected interfaces at compile time.
var _ interface {
@@ -77,13 +72,11 @@ var jwtOptions = []jwt.ParserOption{
jwt.WithLeeway(time.Minute),
}
-var jwtParser = jwt.NewParser(jwtOptions...)
-
-var jwtValidator = jwt.NewValidator(jwtOptions...)
-
// FromData will find and optionally verify JWT secrets in a given set of bytes.
func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {
- client := cmp.Or(s.client, common.SaneHttpClient())
+ jwtParser := jwt.NewParser(jwtOptions...)
+ client := detectors.DetectorHttpClientWithNoLocalAddresses
+
seenMatches := make(map[string]struct{})
for _, matchGroups := range keyPat.FindAllStringSubmatch(string(data), -1) {
@@ -156,23 +149,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (result
return
}
-// Does the URL's refer to a non-routing host?
-func isNonRoutingHost(url *url.URL) bool {
- h := url.Hostname()
- if h == "localhost" {
- return true
- }
-
- ip := net.ParseIP(h)
- if ip != nil {
- return ip.IsPrivate()
- }
-
- return false
-}
-
-// Parse a string into a URL, check that it is an HTTPS URL, and that it doesn't refer to a non-routing host.
-func parseRoutableHttpsUrl(urlString string) (*url.URL, error) {
+// Parse a string into a URL and check that it is an HTTPS URL.
+func parseHttpsUrl(urlString string) (*url.URL, error) {
url, err := url.ParseRequestURI(urlString)
if err != nil {
return nil, err
@@ -180,9 +158,6 @@ func parseRoutableHttpsUrl(urlString string) (*url.URL, error) {
if url.Scheme != "https" {
return nil, fmt.Errorf("only https scheme is supported")
}
- if isNonRoutingHost(url) {
- return nil, fmt.Errorf("only public hosts are supported")
- }
return url, nil
}
@@ -213,6 +188,8 @@ func limitReader(reader io.Reader) io.Reader {
// - If the JWT uses public key cryptography and the OIDC Discovery protocol, we can fetch the public key and perform signature verification
// - In all cases, we can perform claims validation (e.g., checking expiration time) and sometimes get a definite answer that a JWT is *not* live
func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, parsedToken *jwt.Token) (bool, error) {
+ jwtValidator := jwt.NewValidator(jwtOptions...)
+
if err := jwtValidator.Validate(parsedToken.Claims); err != nil {
// though we have not checked the signature, the token is definitely invalid
return false, nil
@@ -225,7 +202,7 @@ func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, pa
// missing or invalid issuer
return false, nil
}
- issuerURL, err := parseRoutableHttpsUrl(issuer)
+ issuerURL, err := parseHttpsUrl(issuer)
if err != nil {
// unsupported issuer
return false, nil
@@ -258,7 +235,7 @@ func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, pa
return false, fmt.Errorf("failed to decode OIDC discovery document: %w", err)
}
- jwksURL, err := parseRoutableHttpsUrl(discoveryDoc.JWKSUri)
+ jwksURL, err := parseHttpsUrl(discoveryDoc.JWKSUri)
if err != nil {
return false, fmt.Errorf("invalid JWKS URL: %w", err)
}
@@ -300,7 +277,7 @@ func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, pa
return false, nil
}
- // signature valid and claims check out
+ // signature valid and claims check out
return true, nil
}