Find, verify, and analyze leaked credentials
Go credentialsdevsecopsdynamic-analysisprecommitscanningsecretsecret-managementsecretssecuritysecurity-toolstrufflehogverification
Add metrics to SaneHTTPClient (#4471)
Amaan Ullah9adec3cc3630
diff --git a/pkg/common/http.go b/pkg/common/http.goindex 87c30fa80..f40612a23 100644--- a/pkg/common/http.go+++ b/pkg/common/http.go@@ -108,6 +108,45 @@ func NewCustomTransport(T http.RoundTripper) *CustomTransport {return &CustomTransport{T}}+type InstrumentedTransport struct {+ T http.RoundTripper+}++func (t *InstrumentedTransport) RoundTrip(req *http.Request) (*http.Response, error) {++ sanitizedURL := sanitizeURL(req.URL.String())++ // increment counter for the URL+ recordHTTPRequest(sanitizedURL)++ // Record start time for latency measurement+ start := time.Now()++ resp, err := t.T.RoundTrip(req)++ // Time the latency+ duration := time.Since(start)++ if err != nil {+ recordNetworkError(sanitizedURL)+ return nil, err+ }++ if resp != nil {+ // record latency and increment counter for non-200 status code+ recordHTTPResponse(sanitizedURL, resp.StatusCode, duration.Seconds())+ }++ return resp, err+}++func NewInstrumentedTransport(T http.RoundTripper) *InstrumentedTransport {+ if T == nil {+ T = http.DefaultTransport+ }+ return &InstrumentedTransport{T}+}+func ConstantResponseHttpClient(statusCode int, body string) *http.Client {return &http.Client{Timeout: DefaultResponseTimeout,@@ -223,7 +262,7 @@ var saneTransport = &http.Transport{func SaneHttpClient() *http.Client {httpClient := &http.Client{}httpClient.Timeout = DefaultResponseTimeout- httpClient.Transport = NewCustomTransport(saneTransport)+ httpClient.Transport = NewInstrumentedTransport(NewCustomTransport(saneTransport))return httpClient}@@ -231,6 +270,6 @@ func SaneHttpClient() *http.Client {func SaneHttpClientTimeOut(timeout time.Duration) *http.Client {httpClient := &http.Client{}httpClient.Timeout = timeout- httpClient.Transport = NewCustomTransport(nil)+ httpClient.Transport = NewInstrumentedTransport(NewCustomTransport(nil))return httpClient}diff --git a/pkg/common/http_metrics.go b/pkg/common/http_metrics.gonew file mode 100644index 000000000..7e0c27287--- /dev/null+++ b/pkg/common/http_metrics.go@@ -0,0 +1,108 @@+package common++import (+ "net/url"+ "strconv"++ "github.com/prometheus/client_golang/prometheus"+ "github.com/prometheus/client_golang/prometheus/promauto"+)++var (+ httpRequestsTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Namespace: MetricsNamespace,+ Subsystem: "http_client",+ Name: "requests_total",+ Help: "Total number of HTTP requests made, labeled by URL.",+ },+ []string{"url"},+ )++ httpRequestDuration = promauto.NewHistogramVec(+ prometheus.HistogramOpts{+ Namespace: MetricsNamespace,+ Subsystem: "http_client",+ Name: "request_duration_seconds",+ Help: "HTTP request latency in seconds, labeled by URL.",+ Buckets: prometheus.DefBuckets,+ },+ []string{"url"},+ )++ httpNon200ResponsesTotal = promauto.NewCounterVec(+ prometheus.CounterOpts{+ Namespace: MetricsNamespace,+ Subsystem: "http_client",+ Name: "non_200_responses_total",+ Help: "Total number of non-200 HTTP responses, labeled by URL and status code.",+ },+ []string{"url", "status_code"},+ )+)++// sanitizeURL sanitizes a URL to avoid high cardinality metrics.+// It keeps only the host and path, removing query parameters, fragments, and user info.+func sanitizeURL(rawURL string) string {+ if rawURL == "" {+ return "unknown"+ }++ parsedURL, err := url.Parse(rawURL)+ if err != nil {+ return "invalid_url"+ }++ // Build sanitized URL with just scheme, host, and path+ sanitized := &url.URL{+ Scheme: parsedURL.Scheme,+ Host: parsedURL.Host,+ Path: parsedURL.Path,+ }++ // If host is empty, try to extract from the raw URL+ if sanitized.Host == "" {+ // For relative URLs or malformed URLs, just use a placeholder+ return "relative_or_invalid"+ }++ // Normalize path+ if sanitized.Path == "" {+ sanitized.Path = "/"+ }++ // Limit path length to avoid extremely long paths creating high cardinality+ if len(sanitized.Path) > 100 {+ sanitized.Path = sanitized.Path[:100] + "..."+ }++ result := sanitized.String()++ // Final fallback to avoid empty strings+ if result == "" {+ return "unknown"+ }++ return result+}++// recordHTTPRequest records metrics for an HTTP request.+func recordHTTPRequest(sanitizedURL string) {+ httpRequestsTotal.WithLabelValues(sanitizedURL).Inc()+}++// recordHTTPResponse records metrics for an HTTP response.+func recordHTTPResponse(sanitizedURL string, statusCode int, durationSeconds float64) {+ // Record latency+ httpRequestDuration.WithLabelValues(sanitizedURL).Observe(durationSeconds)++ // Record non-200 responses+ if statusCode != 200 {+ httpNon200ResponsesTotal.WithLabelValues(sanitizedURL, strconv.Itoa(statusCode)).Inc()+ }+}++// recordNetworkError records metrics for failed HTTP response+func recordNetworkError(sanitizedURL string) {+ httpNon200ResponsesTotal.WithLabelValues(sanitizedURL, "network_error").Inc()+}diff --git a/pkg/common/http_test.go b/pkg/common/http_test.goindex 35df08078..0fb64fd4a 100644--- a/pkg/common/http_test.go+++ b/pkg/common/http_test.go@@ -6,11 +6,14 @@ import ("net/http""net/http/httptest""slices"+ "strings""testing""time""github.com/hashicorp/go-retryablehttp"+ "github.com/prometheus/client_golang/prometheus/testutil""github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require")func TestRetryableHTTPClientCheckRetry(t *testing.T) {@@ -269,3 +272,170 @@ func TestRetryableHTTPClientTimeout(t *testing.T) {})}}++func TestSanitizeURL(t *testing.T) {+ testCases := []struct {+ name string+ input string+ expected string+ }{+ {+ name: "valid https URL",+ input: "https://api.example.com/v1/users",+ expected: "https://api.example.com/v1/users",+ },+ {+ name: "URL with query parameters",+ input: "https://api.example.com/search?q=secret&limit=10",+ expected: "https://api.example.com/search",+ },+ {+ name: "URL with fragment",+ input: "https://example.com/page#section",+ expected: "https://example.com/page",+ },+ {+ name: "URL with user info",+ input: "https://user:[email protected]/path",+ expected: "https://api.example.com/path",+ },+ {+ name: "empty URL",+ input: "",+ expected: "unknown",+ },+ {+ name: "invalid URL",+ input: "not-a-url",+ expected: "relative_or_invalid",+ },+ {+ name: "very long path",+ input: "https://example.com/" + strings.Repeat("a", 150),+ expected: "https://example.com/" + strings.Repeat("a", 99) + "...", // 99 + 1 ("/") = 100 chars+ },+ {+ name: "root path",+ input: "https://example.com",+ expected: "https://example.com/",+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ result := sanitizeURL(tc.input)+ assert.Equal(t, tc.expected, result)+ })+ }+}++func TestSaneHttpClientMetrics(t *testing.T) {+ // Create a test server that returns different status codes+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ switch r.URL.Path {+ case "/success":+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("success"))+ case "/error":+ w.WriteHeader(http.StatusInternalServerError)+ _, _ = w.Write([]byte("error"))+ case "/notfound":+ w.WriteHeader(http.StatusNotFound)+ _, _ = w.Write([]byte("not found"))+ default:+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("default"))+ }+ }))+ defer server.Close()++ // Create a SaneHttpClient+ client := SaneHttpClient()++ testCases := []struct {+ name string+ path string+ expectedStatusCode int+ expectsNon200 bool+ }{+ {+ name: "successful request",+ path: "/success",+ expectedStatusCode: 200,+ expectsNon200: false,+ },+ {+ name: "server error request",+ path: "/error",+ expectedStatusCode: 500,+ expectsNon200: true,+ },+ {+ name: "not found request",+ path: "/notfound",+ expectedStatusCode: 404,+ expectsNon200: true,+ },+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ var requestURL string+ if strings.HasPrefix(tc.path, "http") {+ requestURL = tc.path+ } else {+ requestURL = server.URL + tc.path+ }++ // Get initial metric values+ sanitizedURL := sanitizeURL(requestURL)+ initialRequestsTotal := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL))++ // Make the request+ resp, err := client.Get(requestURL)++ require.NoError(t, err)+ defer resp.Body.Close()+ assert.Equal(t, tc.expectedStatusCode, resp.StatusCode)++ // Check that request counter was incremented+ requestsTotal := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL))+ assert.Equal(t, initialRequestsTotal+1, requestsTotal)+ })+ }+}++func TestInstrumentedTransport(t *testing.T) {+ // Create a mock transport that we can control+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ w.WriteHeader(http.StatusOK)+ _, _ = w.Write([]byte("test response"))+ }))+ defer server.Close()++ // Create instrumented transport+ transport := NewInstrumentedTransport(nil)+ client := &http.Client{+ Transport: transport,+ Timeout: 5 * time.Second,+ }++ // Get initial metric value+ sanitizedURL := sanitizeURL(server.URL)+ initialCount := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL))++ // Make a request+ resp, err := client.Get(server.URL)+ require.NoError(t, err)+ defer resp.Body.Close()++ // Verify the request was successful+ assert.Equal(t, http.StatusOK, resp.StatusCode)++ // Verify metrics were recorded+ finalCount := testutil.ToFloat64(httpRequestsTotal.WithLabelValues(sanitizedURL))+ assert.Equal(t, initialCount+1, finalCount)++ // Note: Testing histogram metrics is complex due to the way Prometheus handles them+ // The main thing is that the request completed successfully and counters were incremented+}