Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Rework JWT detector to better block local IPs; add HTTP instrumentation (#4607)
Brad Larsen728d71fbb3a9
diff --git a/pkg/detectors/detectors.go b/pkg/detectors/detectors.goindex 473e52714..d002db2af 100644--- a/pkg/detectors/detectors.go+++ b/pkg/detectors/detectors.go@@ -18,15 +18,21 @@ import (// Detector defines an interface for scanning for and verifying secrets.type Detector interface {// FromData will scan bytes for results and optionally verify them.+ //+ // FromData can be called concurrently from multiple goroutines.+ // Any modification to the receiver or to global variables will need to to use some kind of synchronization.FromData(ctx context.Context, verify bool, data []byte) ([]Result, error)+// Keywords are used for efficiently pre-filtering chunks using substring operations.// Use unique identifiers that are part of the secret if you can, or the provider name.//// When multiple keywords are provided, they are is treated as a *union* of filtering terms.// That is, if any of the keywords are found in a chunk, the chunk will be run through the detector.Keywords() []string+// Type returns the DetectorType number from detectors.proto for the given detector.Type() detectorspb.DetectorType+// Description returns a description for the result being detectedDescription() string}@@ -93,7 +99,7 @@ type Result struct {// DetectorName is the name of the Detector. Used for custom detectors.DetectorName string// Verified indicates whether the result was verified or not.- Verified bool+ Verified bool// VerificationFromCache indicates whether this result's verification result came from the verification cache rather// than an actual remote request.VerificationFromCache bool@@ -293,7 +299,7 @@ func MustGetBenchmarkData() map[string][]byte {for key, size := range sizes {// Generating a byte slice of a specific size with random data.content := make([]byte, size)- for i := range(size) {+ for i := range size {randomByte, err := rand.Int(rand.Reader, big.NewInt(256))if err != nil {panic(err)diff --git a/pkg/detectors/http.go b/pkg/detectors/http.goindex ed7d21c5d..43ad5052c 100644--- a/pkg/detectors/http.go+++ b/pkg/detectors/http.go@@ -5,9 +5,11 @@ import ("errors""net""net/http"+ "slices""sync""time"+ "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/feature")@@ -44,14 +46,12 @@ var overrideOnce sync.Once// It is guaranteed to only run once, subsequent calls will have no effect.// This should be called before any scans are started.func OverrideDetectorTimeout(timeout time.Duration) {- overrideOnce.Do(func() {- DetectorHttpClientWithLocalAddresses.Timeout = timeout- DetectorHttpClientWithNoLocalAddresses.Timeout = timeout- })+ overrideOnce.Do(func() {+ DetectorHttpClientWithLocalAddresses.Timeout = timeout+ DetectorHttpClientWithNoLocalAddresses.Timeout = timeout+ })}--// ClientOption defines a function type that modifies an http.Client.type ClientOption func(*http.Client)@@ -139,10 +139,8 @@ func WithNoLocalIP() ClientOption {return nil, err}- for _, ip := range ips {- if isLocalIP(ip) {- return nil, ErrNoLocalIP- }+ if slices.ContainsFunc(ips, isLocalIP) {+ return nil, ErrNoLocalIP}return originalDialContext(ctx, network, net.JoinHostPort(host, port))@@ -165,13 +163,15 @@ func WithTimeout(timeout time.Duration) ClientOption {}func NewDetectorHttpClient(opts ...ClientOption) *http.Client {- httpClient := &http.Client{+ client := &http.Client{Transport: NewDetectorTransport(nil),Timeout: DefaultResponseTimeout,}for _, opt := range opts {- opt(httpClient)+ opt(client)}- return httpClient++ client.Transport = common.NewInstrumentedTransport(client.Transport)+ return client}diff --git a/pkg/detectors/jwt/jwt.go b/pkg/detectors/jwt/jwt.goindex f69d5b7ac..03d6e69d6 100644--- a/pkg/detectors/jwt/jwt.go+++ b/pkg/detectors/jwt/jwt.go@@ -1,12 +1,10 @@package jwtimport (- "cmp""context""encoding/json""fmt""io"- "net""net/http""net/url""strings"@@ -17,14 +15,11 @@ import (regexp "github.com/wasilibs/go-re2"- "github.com/trufflesecurity/trufflehog/v3/pkg/common""github.com/trufflesecurity/trufflehog/v3/pkg/detectors""github.com/trufflesecurity/trufflehog/v3/pkg/pb/detectorspb")-type Scanner struct {- client *http.Client-}+type Scanner struct{}// Ensure the Scanner satisfies expected interfaces at compile time.var _ interface {@@ -77,13 +72,11 @@ var jwtOptions = []jwt.ParserOption{jwt.WithLeeway(time.Minute),}-var jwtParser = jwt.NewParser(jwtOptions...)--var jwtValidator = jwt.NewValidator(jwtOptions...)-// FromData will find and optionally verify JWT secrets in a given set of bytes.func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (results []detectors.Result, err error) {- client := cmp.Or(s.client, common.SaneHttpClient())+ jwtParser := jwt.NewParser(jwtOptions...)+ client := detectors.DetectorHttpClientWithNoLocalAddresses+seenMatches := make(map[string]struct{})for _, matchGroups := range keyPat.FindAllStringSubmatch(string(data), -1) {@@ -156,23 +149,8 @@ func (s Scanner) FromData(ctx context.Context, verify bool, data []byte) (resultreturn}-// Does the URL's refer to a non-routing host?-func isNonRoutingHost(url *url.URL) bool {- h := url.Hostname()- if h == "localhost" {- return true- }-- ip := net.ParseIP(h)- if ip != nil {- return ip.IsPrivate()- }-- return false-}--// Parse a string into a URL, check that it is an HTTPS URL, and that it doesn't refer to a non-routing host.-func parseRoutableHttpsUrl(urlString string) (*url.URL, error) {+// Parse a string into a URL and check that it is an HTTPS URL.+func parseHttpsUrl(urlString string) (*url.URL, error) {url, err := url.ParseRequestURI(urlString)if err != nil {return nil, err@@ -180,9 +158,6 @@ func parseRoutableHttpsUrl(urlString string) (*url.URL, error) {if url.Scheme != "https" {return nil, fmt.Errorf("only https scheme is supported")}- if isNonRoutingHost(url) {- return nil, fmt.Errorf("only public hosts are supported")- }return url, nil}@@ -213,6 +188,8 @@ func limitReader(reader io.Reader) io.Reader {// - If the JWT uses public key cryptography and the OIDC Discovery protocol, we can fetch the public key and perform signature verification// - In all cases, we can perform claims validation (e.g., checking expiration time) and sometimes get a definite answer that a JWT is *not* livefunc verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, parsedToken *jwt.Token) (bool, error) {+ jwtValidator := jwt.NewValidator(jwtOptions...)+if err := jwtValidator.Validate(parsedToken.Claims); err != nil {// though we have not checked the signature, the token is definitely invalidreturn false, nil@@ -225,7 +202,7 @@ func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, pa// missing or invalid issuerreturn false, nil}- issuerURL, err := parseRoutableHttpsUrl(issuer)+ issuerURL, err := parseHttpsUrl(issuer)if err != nil {// unsupported issuerreturn false, nil@@ -258,7 +235,7 @@ func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, pareturn false, fmt.Errorf("failed to decode OIDC discovery document: %w", err)}- jwksURL, err := parseRoutableHttpsUrl(discoveryDoc.JWKSUri)+ jwksURL, err := parseHttpsUrl(discoveryDoc.JWKSUri)if err != nil {return false, fmt.Errorf("invalid JWKS URL: %w", err)}@@ -300,7 +277,7 @@ func verifyJWT(ctx context.Context, client *http.Client, tokenParts []string, pareturn false, nil}- // signature valid and claims check out+ // signature valid and claims check outreturn true, nil}