Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
package detectors
import (
"bytes"
"context"
"fmt"
"io"
"net"
"net/http"
"sync"
"time"
"golang.org/x/sync/singleflight"
"github.com/trufflesecurity/trufflehog/v3/pkg/common"
"github.com/trufflesecurity/trufflehog/v3/pkg/feature"
"github.com/trufflesecurity/trufflehog/v3/pkg/ssrf"
)
var DetectorHttpClientWithNoLocalAddresses *http.Client
var DetectorHttpClientWithLocalAddresses *http.Client
// DefaultResponseTimeout is the default timeout for HTTP requests.
const DefaultResponseTimeout = 10 * time.Second
func userAgent() string {
if len(feature.UserAgentSuffix.Load()) > 0 {
return "TruffleHog " + feature.UserAgentSuffix.Load()
}
return "TruffleHog"
}
func init() {
DetectorHttpClientWithLocalAddresses = NewDetectorHttpClient(
WithTransport(NewDetectorTransport(nil)),
WithTimeout(DefaultResponseTimeout),
WithNoFollowRedirects(),
)
DetectorHttpClientWithNoLocalAddresses = NewDetectorHttpClient(
WithTransport(NewDetectorTransport(nil)),
WithTimeout(DefaultResponseTimeout),
WithNoFollowRedirects(),
WithNoLocalIP(),
)
}
var overrideOnce sync.Once
// OverrideDetectorTimeout overrides the default timeout for the detector HTTP clients.
// It is guaranteed to only run once, subsequent calls will have no effect.
// This should be called before any scans are started.
func OverrideDetectorTimeout(timeout time.Duration) {
overrideOnce.Do(func() {
DetectorHttpClientWithLocalAddresses.Timeout = timeout
DetectorHttpClientWithNoLocalAddresses.Timeout = timeout
})
}
// ClientOption defines a function type that modifies an http.Client.
type ClientOption func(*http.Client)
// WithNoFollowRedirects allows disabling automatic following of redirects.
func WithNoFollowRedirects() ClientOption {
return func(c *http.Client) {
c.CheckRedirect = func(req *http.Request, via []*http.Request) error {
return http.ErrUseLastResponse
}
}
}
type detectorTransport struct {
T http.RoundTripper
}
func (t *detectorTransport) RoundTrip(req *http.Request) (*http.Response, error) {
req.Header.Add("User-Agent", userAgent())
return t.T.RoundTrip(req)
}
var defaultDialer = &net.Dialer{
Timeout: 2 * time.Second,
KeepAlive: 5 * time.Second,
}
func NewDetectorTransport(T http.RoundTripper) http.RoundTripper {
if T == nil {
T = &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: defaultDialer.DialContext,
MaxIdleConns: 100,
MaxIdleConnsPerHost: 5,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 3 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
}
}
return &detectorTransport{T: T}
}
// ErrNoLocalIP is returned when a client configured with WithNoLocalIP
// refuses to dial a non-public address.
//
// Deprecated: it is an alias of ssrf.ErrEgressBlocked; new code should use
// that sentinel directly.
var ErrNoLocalIP = ssrf.ErrEgressBlocked
// WithNoLocalIP configures the client to refuse connections to non-public
// addresses: loopback, link-local (incl. cloud metadata), private, CGNAT,
// multicast, and the other special-use ranges the ssrf package classifies.
// The check runs in the guarded dialer on each resolved address, so it is
// DNS-rebinding safe (the previous implementation vetted a LookupIP result
// and then dialed the hostname again, which a rebinding resolver could race)
// and it re-runs on every redirect hop's fresh dial.
//
// Two behavior notes against the previous implementation: any DialContext
// already set on the transport is REPLACED, not chained, so custom dialers
// (SOCKS, custom resolvers) are discarded; and blocking is per resolved
// address, so a hostname resolving to both a public and a non-public record
// connects via the public one where it previously refused the whole host.
func WithNoLocalIP() ClientOption {
return func(c *http.Client) {
if c.Transport == nil {
c.Transport = &http.Transport{}
}
// Type assertion to get the underlying *http.Transport
transport, ok := c.Transport.(*http.Transport)
if !ok {
// If c.Transport is not *http.Transport, check if it is wrapped in a detectorTransport
dt, ok := c.Transport.(*detectorTransport)
if !ok {
panic("unsupported transport type")
}
transport, ok = dt.T.(*http.Transport)
if !ok {
panic("underlying transport is not *http.Transport")
}
}
// The guarded dialer replaces any existing DialContext; within this
// package that is only ever nil or defaultDialer, whose settings the
// guard preserves.
transport.DialContext = ssrf.GuardDialer(defaultDialer).DialContext
}
}
// WithTransport sets a custom transport for the http.Client.
func WithTransport(transport http.RoundTripper) ClientOption {
return func(c *http.Client) {
c.Transport = transport
}
}
// WithTimeout sets a timeout for the http.Client.
func WithTimeout(timeout time.Duration) ClientOption {
return func(c *http.Client) {
c.Timeout = timeout
}
}
func NewDetectorHttpClient(opts ...ClientOption) *http.Client {
client := &http.Client{
Transport: NewDetectorTransport(nil),
Timeout: DefaultResponseTimeout,
}
for _, opt := range opts {
opt(client)
}
client.Transport = common.NewInstrumentedTransport(client.Transport)
return client
}
// bufferedResponse holds a fully-read HTTP response so it can be replayed to
// every goroutine that was coalesced by singleflight.
type bufferedResponse struct {
statusCode int
header http.Header
body []byte
}
// singleflightTransport is an http.RoundTripper that coalesces concurrent requests
// sharing the same deduplication key into a single network call. It is a no-op for
// requests whose context does not carry a dedup key.
type singleflightTransport struct {
base http.RoundTripper
group singleflight.Group
}
func (t *singleflightTransport) RoundTrip(req *http.Request) (*http.Response, error) {
key, ok := req.Context().Value(dedupKeyContextKey{}).(string)
if !ok || key == "" {
return t.base.RoundTrip(req)
}
// DoChan is used instead of Do so each caller can independently respect its
// own context cancellation without blocking on the shared in-flight call.
ch := t.group.DoChan(key, func() (any, error) {
// Detach the in-flight request from the first caller's cancellation so
// that one goroutine timing out doesn't abort the shared network call
// and propagate an error to all coalesced waiters.
//
// context.WithoutCancel also strips any deadline (e.g. from
// http.Client.Timeout), so we re-attach the original deadline if
// present. Without this the shared request has no timeout and a
// hanging server would leak the goroutine and pin the singleflight
// key indefinitely.
sharedCtx := context.WithoutCancel(req.Context())
if deadline, ok := req.Context().Deadline(); ok {
var cancel context.CancelFunc
sharedCtx, cancel = context.WithDeadline(sharedCtx, deadline)
defer cancel()
}
sharedReq := req.WithContext(sharedCtx)
resp, err := t.base.RoundTrip(sharedReq)
if err != nil {
return nil, err
}
defer func() { _ = resp.Body.Close() }()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, err
}
return &bufferedResponse{
statusCode: resp.StatusCode,
header: resp.Header.Clone(),
body: body,
}, nil
})
select {
case result := <-ch:
if result.Err != nil {
return nil, result.Err
}
br := result.Val.(*bufferedResponse)
return &http.Response{
StatusCode: br.statusCode,
Status: fmt.Sprintf("%d %s", br.statusCode, http.StatusText(br.statusCode)),
Header: br.header.Clone(),
Body: io.NopCloser(bytes.NewReader(br.body)),
}, nil
case <-req.Context().Done():
return nil, req.Context().Err()
}
}
// NewClientWithDedup wraps base with a transport that deduplicates concurrent
// verification requests sharing the same key. Detectors opt in per credential by
// calling WithDedupKey on the request context before client.Do — no other changes
// to request building or response reading are needed.
func NewClientWithDedup(base *http.Client) *http.Client {
clone := *base
transport := base.Transport
if transport == nil {
transport = http.DefaultTransport
}
clone.Transport = &singleflightTransport{base: transport}
return &clone
}