Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cyclops): break down latency alerts by path and user (#6832)
r33drichards73c1d7813a4e
diff --git a/libs/fleet/backend/auth/middlewares.go b/libs/fleet/backend/auth/middlewares.goindex 804311c75c..cb76693894 100644--- a/libs/fleet/backend/auth/middlewares.go+++ b/libs/fleet/backend/auth/middlewares.go@@ -12,6 +12,7 @@ import ("sync""time"+ "cyclops-cs-backend/metrics""cyclops-cs-backend/middlewares""github.com/open-feature/go-sdk/openfeature"@@ -383,6 +384,7 @@ func TokenAuthMiddleware(next http.Handler) http.Handler {Email: proxyEmail,AZP: "oauth2-proxy", // distinct from SPA/key clients for OPA}+ metrics.SetRequestUser(r.Context(), user.ID)next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), UserKey, user)))return}@@ -418,6 +420,7 @@ func TokenAuthMiddleware(next http.Handler) http.Handler {return}+ metrics.SetRequestUser(r.Context(), user.ID)next.ServeHTTP(w, r.WithContext(context.WithValue(r.Context(), UserKey, user)))})}diff --git a/libs/fleet/backend/metrics/metrics.go b/libs/fleet/backend/metrics/metrics.goindex f5abfcef82..0a16e66141 100644--- a/libs/fleet/backend/metrics/metrics.go+++ b/libs/fleet/backend/metrics/metrics.go@@ -3,7 +3,7 @@// Exported metrics://// cyclops_cs_http_requests_total - counter, labels: method, path, status_code-// cyclops_cs_http_request_duration_seconds - histogram, labels: method, path, status_code+// cyclops_cs_http_request_duration_seconds - histogram, labels: method, path, status_code, user// cyclops_cs_keycloak_requests_total - counter, labels: operation, status// cyclops_cs_keycloak_request_duration_seconds - histogram, labels: operation, status// cyclops_cs_upstream_proxy_requests_total - counter, labels: target, status_code@@ -16,6 +16,7 @@ package metricsimport ("bufio"+ "context""fmt""net""net/http"@@ -38,7 +39,7 @@ var (Name: "cyclops_cs_http_request_duration_seconds",Help: "Latency of HTTP requests handled by the cyclops-cs backend.",Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10},- }, []string{"method", "path", "status_code"})+ }, []string{"method", "path", "status_code", "user"})ActiveRequests = promauto.NewGauge(prometheus.GaugeOpts{Name: "cyclops_cs_active_requests",@@ -128,6 +129,19 @@ func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {return hj.Hijack()}+type requestLabelsKey struct{}++type requestLabels struct {+ user string+}++// SetRequestUser attaches the authenticated subject to the current request metrics.+func SetRequestUser(ctx context.Context, user string) {+ if labels, ok := ctx.Value(requestLabelsKey{}).(*requestLabels); ok && user != "" {+ labels.user = user+ }+}+// Middleware returns an http.Handler that records HTTP SLI metrics for// each request. It must be placed as the outermost handler wrapper so// that it captures the final status code from all inner layers (auth,@@ -135,6 +149,8 @@ func (rw *responseWriter) Hijack() (net.Conn, *bufio.ReadWriter, error) {func Middleware(next http.Handler) http.Handler {return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {start := time.Now()+ labels := &requestLabels{user: "unknown"}+ r = r.WithContext(context.WithValue(r.Context(), requestLabelsKey{}, labels))rw := newResponseWriter(w)ActiveRequests.Inc()defer ActiveRequests.Dec()@@ -146,7 +162,7 @@ func Middleware(next http.Handler) http.Handler {path := normalizePath(r.URL.Path)HTTPRequestsTotal.WithLabelValues(r.Method, path, statusStr).Inc()- HTTPRequestDuration.WithLabelValues(r.Method, path, statusStr).Observe(duration)+ HTTPRequestDuration.WithLabelValues(r.Method, path, statusStr, labels.user).Observe(duration)})}diff --git a/libs/fleet/backend/metrics/metrics_test.go b/libs/fleet/backend/metrics/metrics_test.goindex 98dcb53ec1..31b3327e78 100644--- a/libs/fleet/backend/metrics/metrics_test.go+++ b/libs/fleet/backend/metrics/metrics_test.go@@ -1,9 +1,69 @@package metricsimport (+ "net/http"+ "net/http/httptest""testing"++ "github.com/prometheus/client_golang/prometheus")+func TestMiddlewareRecordsRequestUserOnLatencyHistogram(t *testing.T) {+ handler := Middleware(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ SetRequestUser(r.Context(), "user-123")+ w.WriteHeader(http.StatusNoContent)+ }))+ handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/api/keys", nil))++ if count := histogramCount(t, map[string]string{+ "method": "GET",+ "path": "/api/keys",+ "status_code": "204",+ "user": "user-123",+ }); count != 1 {+ t.Fatalf("request histogram count = %d, want 1", count)+ }+}++func TestMiddlewareRecordsUnknownUserByDefault(t *testing.T) {+ handler := Middleware(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {+ w.WriteHeader(http.StatusOK)+ }))+ handler.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest(http.MethodGet, "/healthz", nil))++ if count := histogramCount(t, map[string]string{+ "method": "GET",+ "path": "/healthz",+ "status_code": "200",+ "user": "unknown",+ }); count != 1 {+ t.Fatalf("request histogram count = %d, want 1", count)+ }+}++func histogramCount(t *testing.T, labels map[string]string) uint64 {+ t.Helper()+ families, err := prometheus.DefaultGatherer.Gather()+ if err != nil {+ t.Fatal(err)+ }+ for _, family := range families {+ if family.GetName() != "cyclops_cs_http_request_duration_seconds" {+ continue+ }+ for _, metric := range family.Metric {+ matched := len(metric.Label) == len(labels)+ for _, pair := range metric.Label {+ matched = matched && labels[pair.GetName()] == pair.GetValue()+ }+ if matched {+ return metric.GetHistogram().GetSampleCount()+ }+ }+ }+ return 0+}+func TestNormalizePath(t *testing.T) {cases := []struct {in string