Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cyclops): break down latency alerts by path and user (#6832)

r33drichards73c1d7813a4e
diff --git a/libs/fleet/backend/auth/middlewares.go b/libs/fleet/backend/auth/middlewares.go
index 804311c75c..cb76693894 100644
--- a/libs/fleet/backend/auth/middlewares.go
+++ b/libs/fleet/backend/auth/middlewares.go
@@ -12,6 +12,7 @@ import (
"sync"
"time"
+ "cyclops-cs-backend/metrics"
"cyclops-cs-backend/middlewares"
"github.com/open-feature/go-sdk/openfeature"
@@ -383,6 +384,7 @@ func TokenAuthMiddleware(next http.Handler) http.Handler {
Email: proxyEmail,
AZP: "oauth2-proxy", // distinct from SPA/key clients for OPA
}
+ metrics.SetRequestUser(r.Context(), user.ID)
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), UserKey, user)))
return
}
@@ -418,6 +420,7 @@ func TokenAuthMiddleware(next http.Handler) http.Handler {
return
}
+ metrics.SetRequestUser(r.Context(), user.ID)
next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), UserKey, user)))
})
}
diff --git a/libs/fleet/backend/metrics/metrics.go b/libs/fleet/backend/metrics/metrics.go
index f5abfcef82..0a16e66141 100644
--- a/libs/fleet/backend/metrics/metrics.go
+++ b/libs/fleet/backend/metrics/metrics.go
@@ -3,7 +3,7 @@
// Exported metrics:
//
// cyclops_cs_http_requests_total - counter, labels: method, path, status_code
-// cyclops_cs_http_request_duration_seconds - histogram, labels: method, path, status_code
+// cyclops_cs_http_request_duration_seconds - histogram, labels: method, path, status_code, user
// cyclops_cs_keycloak_requests_total - counter, labels: operation, status
// cyclops_cs_keycloak_request_duration_seconds - histogram, labels: operation, status
// cyclops_cs_upstream_proxy_requests_total - counter, labels: target, status_code
@@ -16,6 +16,7 @@ package metrics
import (
"bufio"
+ "context"
"fmt"
"net"
"net/http"
@@ -38,7 +39,7 @@ var (
Name: "cyclops_cs_http_request_duration_seconds",
Help: "Latency of HTTP requests handled by the cyclops-cs backend.",
Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},
- }, []string{"method", "path", "status_code"})
+ }, []string{"method", "path", "status_code", "user"})
ActiveRequests = promauto.NewGauge(prometheus.GaugeOpts{
Name: "cyclops_cs_active_requests",
@@ -128,6 +129,19 @@ func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
return hj.Hijack()
}
+type requestLabelsKey struct{}
+
+type requestLabels struct {
+ user string
+}
+
+// SetRequestUser attaches the authenticated subject to the current request metrics.
+func SetRequestUser(ctx context.Context, user string) {
+ if labels, ok := ctx.Value(requestLabelsKey{}).(*requestLabels); ok && user != "" {
+ labels.user = user
+ }
+}
+
// Middleware returns an http.Handler that records HTTP SLI metrics for
// each request. It must be placed as the outermost handler wrapper so
// that it captures the final status code from all inner layers (auth,
@@ -135,6 +149,8 @@ func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {
func Middleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
+ labels := &requestLabels{user: "unknown"}
+ r = r.WithContext(context.WithValue(r.Context(), requestLabelsKey{}, labels))
rw := newResponseWriter(w)
ActiveRequests.Inc()
defer ActiveRequests.Dec()
@@ -146,7 +162,7 @@ func Middleware(next http.Handler) http.Handler {
path := normalizePath(r.URL.Path)
HTTPRequestsTotal.WithLabelValues(r.Method, path, statusStr).Inc()
- HTTPRequestDuration.WithLabelValues(r.Method, path, statusStr).Observe(duration)
+ HTTPRequestDuration.WithLabelValues(r.Method, path, statusStr, labels.user).Observe(duration)
})
}
diff --git a/libs/fleet/backend/metrics/metrics_test.go b/libs/fleet/backend/metrics/metrics_test.go
index 98dcb53ec1..31b3327e78 100644
--- a/libs/fleet/backend/metrics/metrics_test.go
+++ b/libs/fleet/backend/metrics/metrics_test.go
@@ -1,9 +1,69 @@
package metrics
import (
+ "net/http"
+ "net/http/httptest"
"testing"
+
+ "github.com/prometheus/client_golang/prometheus"
)
+func TestMiddlewareRecordsRequestUserOnLatencyHistogram(t *testing.T) {
+ handler := Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ SetRequestUser(r.Context(), "user-123")
+ w.WriteHeader(http.StatusNoContent)
+ }))
+ handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/keys", nil))
+
+ if count := histogramCount(t, map[string]string{
+ "method": "GET",
+ "path": "/api/keys",
+ "status_code": "204",
+ "user": "user-123",
+ }); count != 1 {
+ t.Fatalf("request histogram count = %d, want 1", count)
+ }
+}
+
+func TestMiddlewareRecordsUnknownUserByDefault(t *testing.T) {
+ handler := Middleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+ w.WriteHeader(http.StatusOK)
+ }))
+ handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/healthz", nil))
+
+ if count := histogramCount(t, map[string]string{
+ "method": "GET",
+ "path": "/healthz",
+ "status_code": "200",
+ "user": "unknown",
+ }); count != 1 {
+ t.Fatalf("request histogram count = %d, want 1", count)
+ }
+}
+
+func histogramCount(t *testing.T, labels map[string]string) uint64 {
+ t.Helper()
+ families, err := prometheus.DefaultGatherer.Gather()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, family := range families {
+ if family.GetName() != "cyclops_cs_http_request_duration_seconds" {
+ continue
+ }
+ for _, metric := range family.Metric {
+ matched := len(metric.Label) == len(labels)
+ for _, pair := range metric.Label {
+ matched = matched && labels[pair.GetName()] == pair.GetValue()
+ }
+ if matched {
+ return metric.GetHistogram().GetSampleCount()
+ }
+ }
+ }
+ return 0
+}
+
func TestNormalizePath(t *testing.T) {
cases := []struct {
in string