Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cyclops): trace usage latency breakdown (#7118)
r33drichards3d797367152b
diff --git a/libs/fleet/backend/handlers/usage.go b/libs/fleet/backend/handlers/usage.goindex 0613fbfc93..d4a9155a3e 100644--- a/libs/fleet/backend/handlers/usage.go+++ b/libs/fleet/backend/handlers/usage.go@@ -8,6 +8,9 @@ import ("strings""cyclops-cs-backend/usage"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/codes"+ "go.opentelemetry.io/otel/trace")type UsageTimeframe = usage.Timeframe@@ -45,6 +48,9 @@ func parseUsageTimeframe(raw string) (UsageTimeframe, UsageInterval, bool) {}func (h Handlers) authorizeUsage(w http.ResponseWriter, r *http.Request) (UsageQuery, UsageInterval, bool) {+ ctx, span := handlerTracer().Start(r.Context(), "usage.authorize")+ defer span.End()+u := currentUser(r)if u == nil || u.ID == "" {writeErr(w, 401, "authentication required")@@ -55,7 +61,7 @@ func (h Handlers) authorizeUsage(w http.ResponseWriter, r *http.Request) (UsageQwriteErr(w, 400, "timeframe must be one of 24h, 7d, or 30d")return UsageQuery{}, "", false}- enabled, err := h.usageEnabled(r.Context(), u)+ enabled, err := h.usageEnabled(ctx, u)if err != nil || !enabled {if err != nil {slog.WarnContext(r.Context(), "usage access evaluation failed", "err", err)@@ -63,7 +69,7 @@ func (h Handlers) authorizeUsage(w http.ResponseWriter, r *http.Request) (UsageQwriteErr(w, 403, "usage preview is not enabled")return UsageQuery{}, "", false}- admin, err := h.isAdmin(r.Context(), u)+ admin, err := h.isAdmin(ctx, u)if err != nil {admin = false}@@ -78,10 +84,20 @@ func (h Handlers) authorizeUsage(w http.ResponseWriter, r *http.Request) (UsageQwriteErr(w, 400, "subject is too long")return UsageQuery{}, "", false}+ span.SetAttributes(+ attribute.String("usage.timeframe", string(tf)),+ attribute.String("usage.interval", string(iv)),+ attribute.Bool("usage.admin", admin),+ attribute.Bool("usage.subject_override", sub != u.ID),+ attribute.Bool("usage.authorized", true),+ )return UsageQuery{ActorSubject: u.ID, Subject: sub, Admin: admin, Timeframe: tf}, iv, true}func (h Handlers) GetUsageOverview(w http.ResponseWriter, r *http.Request) {+ ctx, span := handlerTracer().Start(r.Context(), "usage.overview")+ defer span.End()+ r = r.WithContext(ctx)w.Header().Set("Cache-Control", "private, no-store")q, _, ok := h.authorizeUsage(w, r)if !ok {@@ -91,8 +107,10 @@ func (h Handlers) GetUsageOverview(w http.ResponseWriter, r *http.Request) {writeErr(w, 503, "usage data provider is not configured")return}+ span.SetAttributes(attribute.String("usage.timeframe", string(q.Timeframe)))v, err := h.Usage.Overview(r.Context(), q)if err != nil {+ markUsageHandlerError(span, "usage overview failed")writeErr(w, 502, "usage data is temporarily unavailable")return}@@ -100,6 +118,9 @@ func (h Handlers) GetUsageOverview(w http.ResponseWriter, r *http.Request) {}func (h Handlers) GetUsagePoolDetail(w http.ResponseWriter, r *http.Request) {+ ctx, span := handlerTracer().Start(r.Context(), "usage.pool_detail")+ defer span.End()+ r = r.WithContext(ctx)w.Header().Set("Cache-Control", "private, no-store")q, iv, ok := h.authorizeUsage(w, r)if !ok {@@ -114,14 +135,23 @@ func (h Handlers) GetUsagePoolDetail(w http.ResponseWriter, r *http.Request) {writeErr(w, 503, "usage data provider is not configured")return}+ span.SetAttributes(+ attribute.String("usage.timeframe", string(q.Timeframe)),+ attribute.String("usage.interval", string(iv)),+ )v, err := h.Usage.PoolDetail(r.Context(), UsagePoolQuery{Query: q, PoolID: pool, Interval: iv})if errors.Is(err, usage.ErrPoolNotFound) {writeErr(w, 404, "usage pool was not found")return}if err != nil {+ markUsageHandlerError(span, "usage pool detail failed")writeErr(w, 502, "usage data is temporarily unavailable")return}writeJSON(w, 200, v)}++func markUsageHandlerError(span trace.Span, description string) {+ span.SetStatus(codes.Error, description)+}diff --git a/libs/fleet/backend/handlers/usage_test.go b/libs/fleet/backend/handlers/usage_test.goindex fdf1306222..0802d3f9e5 100644--- a/libs/fleet/backend/handlers/usage_test.go+++ b/libs/fleet/backend/handlers/usage_test.go@@ -8,6 +8,10 @@ import ("net/http""net/http/httptest""testing"++ "go.opentelemetry.io/otel"+ sdktrace "go.opentelemetry.io/otel/sdk/trace"+ "go.opentelemetry.io/otel/sdk/trace/tracetest")type fakeUsageProvider struct {@@ -105,3 +109,46 @@ func TestUsageProviderErrorsAreRedacted(t *testing.T) {}})}++func TestUsagePoolDetailCreatesHandlerAndAuthorizationSpans(t *testing.T) {+ recorder := tracetest.NewSpanRecorder()+ provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))+ previousProvider := otel.GetTracerProvider()+ otel.SetTracerProvider(provider)+ t.Cleanup(func() { otel.SetTracerProvider(previousProvider) })++ ctx, root := provider.Tracer("test").Start(context.Background(), "request")+ request := withUser(+ httptest.NewRequest(http.MethodGet, "/api/usage/pool?timeframe=24h&pool=pool-should-not-appear", nil).WithContext(ctx),+ &auth.User{ID: "user-should-not-appear"},+ )+ response := httptest.NewRecorder()++ uh(&fakeUsageProvider{}, true, false).GetUsagePoolDetail(response, request)+ root.End()++ if response.Code != http.StatusOK {+ t.Fatalf("status = %d, want %d", response.Code, http.StatusOK)+ }+ for _, name := range []string{"usage.pool_detail", "usage.authorize"} {+ if handlerSpanByName(recorder.Ended(), name) == nil {+ t.Fatalf("expected %q span", name)+ }+ }+ for _, span := range recorder.Ended() {+ for _, attr := range span.Attributes() {+ if attr.Value.AsString() == "user-should-not-appear" || attr.Value.AsString() == "pool-should-not-appear" {+ t.Fatalf("span %q contains sensitive/high-cardinality attribute %q", span.Name(), attr.Key)+ }+ }+ }+}++func handlerSpanByName(spans []sdktrace.ReadOnlySpan, name string) sdktrace.ReadOnlySpan {+ for _, span := range spans {+ if span.Name() == name {+ return span+ }+ }+ return nil+}diff --git a/libs/fleet/backend/usage/provider.go b/libs/fleet/backend/usage/provider.goindex 69e87ded38..d48333d636 100644--- a/libs/fleet/backend/usage/provider.go+++ b/libs/fleet/backend/usage/provider.go@@ -9,6 +9,8 @@ import ("time""cyclops-cs-backend/identity"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/trace")var ErrPoolNotFound = errors.New("usage pool not found")@@ -65,12 +67,17 @@ func (p *provider) PoolDetail(ctx context.Context, query PoolQuery) (PoolDetailRif !ok {return PoolDetailResponse{}, fmt.Errorf("%w: %s", ErrPoolNotFound, query.PoolID)}+ _, bucketSpan := usageTracer().Start(ctx, "usage.buckets.build", trace.WithAttributes(+ attribute.String("usage.interval", string(query.Interval)),+ ))response := PoolDetailResponse{DataAsOf: data.asOf, Partial: data.partial, Pool: PoolIdentity{ID: pool.id, Name: pool.name}, Buckets: make([]Bucket, 0)}for start := data.start; start.Before(data.end); start = start.Add(step) {key := bucketKey{namespace: namespace, pool: poolName, start: start}totals := data.buckets[key]response.Buckets = append(response.Buckets, Bucket{Start: start, End: start.Add(step), CPUConsumed: totals.cpuConsumed, CPUProvisioned: totals.cpuProvisioned, MemoryConsumed: totals.memoryConsumed, MemoryProvisioned: totals.memoryProvisioned})}+ bucketSpan.SetAttributes(attribute.Int("usage.bucket_count", len(response.Buckets)))+ bucketSpan.End()return response, nil}@@ -99,26 +106,74 @@ type bucketKey struct {}func (p *provider) load(ctx context.Context, query Query) (usageData, error) {+ ctx, span := usageTracer().Start(ctx, "usage.load", trace.WithAttributes(+ attribute.String("usage.timeframe", string(query.Timeframe)),+ attribute.Bool("usage.admin", query.Admin),+ attribute.Bool("usage.subject_override", query.ActorSubject != "" && query.ActorSubject != query.Subject),+ ))+ defer span.End()+if p.events == nil || p.allocations == nil {- return usageData{}, fmt.Errorf("usage provider is not configured")+ err := fmt.Errorf("usage provider is not configured")+ markUsageSpanError(span, "usage provider unavailable")+ return usageData{}, err}start, end, step, err := usageWindow(query.Timeframe, p.clock())if err != nil {+ markUsageSpanError(span, "invalid usage window")return usageData{}, err}- events, err := p.events.Events(ctx, identity.PersonalGroup(ctx, query.Subject), start, end)+ span.SetAttributes(attribute.Int64("usage.window_seconds", int64(end.Sub(start).Seconds())))++ eventsCtx, eventsSpan := usageTracer().Start(ctx, "usage.events.query")+ events, err := p.events.Events(eventsCtx, identity.PersonalGroup(eventsCtx, query.Subject), start, end)if err != nil {+ markUsageSpanError(eventsSpan, "usage event query failed")+ eventsSpan.End()+ markUsageSpanError(span, "usage event query failed")return usageData{}, fmt.Errorf("read usage events: %w", err)}+ eventsSpan.SetAttributes(attribute.Int("usage.event_count", len(events)))+ eventsSpan.End()++ _, segmentsSpan := usageTracer().Start(ctx, "usage.segments.build", trace.WithAttributes(+ attribute.Int("usage.event_count", len(events)),+ ))segments, namespaces, partial := buildSegments(events, start, end)+ segmentsSpan.SetAttributes(+ attribute.Int("usage.segment_count", len(segments)),+ attribute.Int("usage.namespace_count", len(namespaces)),+ attribute.Bool("usage.partial", partial),+ )+ segmentsSpan.End()+data := usageData{start: start, end: end, asOf: end, partial: partial, pools: make([]usageTotals, 0), buckets: make(map[bucketKey]usageTotals)}if len(segments) == 0 || len(namespaces) == 0 {+ span.SetAttributes(+ attribute.Int("usage.pool_count", 0),+ attribute.Int("usage.bucket_count", 0),+ attribute.Bool("usage.partial", data.partial),+ )return data, nil}- allocations, asOf, sourcePartial, err := p.allocations.Allocations(ctx, start, end, step, namespaces)++ allocationsCtx, allocationsSpan := usageTracer().Start(ctx, "usage.allocations.query", trace.WithAttributes(+ attribute.Int("usage.namespace_count", len(namespaces)),+ attribute.Int64("usage.step_seconds", int64(step.Seconds())),+ ))+ allocations, asOf, sourcePartial, err := p.allocations.Allocations(allocationsCtx, start, end, step, namespaces)if err != nil {+ markUsageSpanError(allocationsSpan, "OpenCost allocation query failed")+ allocationsSpan.End()+ markUsageSpanError(span, "OpenCost allocation query failed")return usageData{}, fmt.Errorf("read OpenCost allocations: %w", err)}+ allocationsSpan.SetAttributes(+ attribute.Int("usage.allocation_count", len(allocations)),+ attribute.Bool("usage.partial", sourcePartial),+ )+ allocationsSpan.End()+if asOf.IsZero() {asOf = startsourcePartial = true@@ -129,6 +184,10 @@ func (p *provider) load(ctx context.Context, query Query) (usageData, error) {sourcePartial = true}data.partial = data.partial || sourcePartial || data.asOf.Before(end)++ _, attributionSpan := usageTracer().Start(ctx, "usage.allocations.attribute", trace.WithAttributes(+ attribute.Int("usage.allocation_count", len(allocations)),+ ))poolTotals := make(map[string]usageTotals)for _, allocation := range allocations {matched, allocationPartial := attributeAllocation(allocation, segments, start, end, step, poolTotals, data.buckets)@@ -143,6 +202,17 @@ func (p *provider) load(ctx context.Context, query Query) (usageData, error) {}disambiguatePoolNames(data.pools)sort.Slice(data.pools, func(i, j int) bool { return data.pools[i].id < data.pools[j].id })+ attributionSpan.SetAttributes(+ attribute.Int("usage.pool_count", len(data.pools)),+ attribute.Int("usage.bucket_count", len(data.buckets)),+ attribute.Bool("usage.partial", data.partial),+ )+ attributionSpan.End()+ span.SetAttributes(+ attribute.Int("usage.pool_count", len(data.pools)),+ attribute.Int("usage.bucket_count", len(data.buckets)),+ attribute.Bool("usage.partial", data.partial),+ )return data, nil}diff --git a/libs/fleet/backend/usage/tracing.go b/libs/fleet/backend/usage/tracing.gonew file mode 100644index 0000000000..7ff0715284--- /dev/null+++ b/libs/fleet/backend/usage/tracing.go@@ -0,0 +1,15 @@+package usage++import (+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/codes"+ "go.opentelemetry.io/otel/trace"+)++func usageTracer() trace.Tracer {+ return otel.Tracer("cyclops-cs-backend/usage")+}++func markUsageSpanError(span trace.Span, description string) {+ span.SetStatus(codes.Error, description)+}diff --git a/libs/fleet/backend/usage/tracing_test.go b/libs/fleet/backend/usage/tracing_test.gonew file mode 100644index 0000000000..8b8662376d--- /dev/null+++ b/libs/fleet/backend/usage/tracing_test.go@@ -0,0 +1,132 @@+package usage++import (+ "context"+ "errors"+ "fmt"+ "strings"+ "testing"+ "time"++ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/codes"+ sdktrace "go.opentelemetry.io/otel/sdk/trace"+ "go.opentelemetry.io/otel/sdk/trace/tracetest"+)++func TestProviderPoolDetailCreatesLatencyBreakdownSpans(t *testing.T) {+ cutoff := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC)+ recorder := tracetest.NewSpanRecorder()+ provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))+ previousProvider := otel.GetTracerProvider()+ otel.SetTracerProvider(provider)+ t.Cleanup(func() { otel.SetTracerProvider(previousProvider) })++ ctx, root := provider.Tracer("test").Start(context.Background(), "request")+ usageProvider := NewProvider(+ &fakeEventStore{events: []SandboxEvent{{+ EventID: mustUUID("00000000-0000-0000-0000-000000000101"), Namespace: "ns-a", SandboxName: "sandbox-a", SandboxUID: "uid-a",+ PoolName: "pool-a", Runtime: "kubevirt", VMName: "vm-a", EventType: "Added", ObservedAt: cutoff.Add(-25 * time.Hour),+ }}},+ &fakeAllocationClient{asOf: cutoff, allocations: []Allocation{{+ Start: cutoff.Add(-time.Hour), End: cutoff, Namespace: "ns-a", Pod: "virt-launcher-vm-a-abc",+ Minutes: 60, CPUUsageAverage: 2, CPURequestAverage: 4,+ RAMUsageAverageBytes: 3 * gibibyte, RAMRequestAverageBytes: 6 * gibibyte,+ }}},+ func() time.Time { return cutoff },+ )++ _, err := usageProvider.PoolDetail(ctx, PoolQuery{+ Query: Query{Subject: "user-should-not-appear", Timeframe: Timeframe24H},+ PoolID: "ns-a:pool-a",+ Interval: IntervalHour,+ })+ root.End()+ if err != nil {+ t.Fatal(err)+ }++ for _, name := range []string{+ "usage.load",+ "usage.events.query",+ "usage.segments.build",+ "usage.allocations.query",+ "usage.allocations.attribute",+ "usage.buckets.build",+ } {+ if findEndedSpan(recorder.Ended(), name) == nil {+ t.Fatalf("expected %q span", name)+ }+ }++ loadSpan := findEndedSpan(recorder.Ended(), "usage.load")+ assertSpanAttribute(t, loadSpan.Attributes(), "usage.timeframe", "24h")+ assertSpanAttribute(t, findEndedSpan(recorder.Ended(), "usage.events.query").Attributes(), "usage.event_count", int64(1))+ assertSpanAttribute(t, findEndedSpan(recorder.Ended(), "usage.allocations.query").Attributes(), "usage.allocation_count", int64(1))+ assertSpanAttribute(t, findEndedSpan(recorder.Ended(), "usage.buckets.build").Attributes(), "usage.bucket_count", int64(24))++ for _, span := range recorder.Ended() {+ for _, attr := range span.Attributes() {+ if attr.Value.AsString() == "user-should-not-appear" || attr.Value.AsString() == "ns-a:pool-a" {+ t.Fatalf("span %q contains sensitive/high-cardinality attribute %q", span.Name(), attr.Key)+ }+ }+ }+}++func findEndedSpan(spans []sdktrace.ReadOnlySpan, name string) sdktrace.ReadOnlySpan {+ for _, span := range spans {+ if span.Name() == name {+ return span+ }+ }+ return nil+}++func assertSpanAttribute(t *testing.T, attributes []attribute.KeyValue, key string, want any) {+ t.Helper()+ for _, attr := range attributes {+ if string(attr.Key) == key {+ if got := attr.Value.AsInterface(); got != want {+ t.Fatalf("attribute %s = %#v, want %#v", key, got, want)+ }+ return+ }+ }+ t.Fatalf("missing attribute %s", key)+}++func TestProviderErrorSpansDoNotExportSourceMessages(t *testing.T) {+ recorder := tracetest.NewSpanRecorder()+ provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))+ previousProvider := otel.GetTracerProvider()+ otel.SetTracerProvider(provider)+ t.Cleanup(func() { otel.SetTracerProvider(previousProvider) })++ ctx, root := provider.Tracer("test").Start(context.Background(), "request")+ _, err := NewProvider(+ &fakeEventStore{err: errors.New("postgres://[email protected]: source unavailable")},+ &fakeAllocationClient{},+ func() time.Time { return time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) },+ ).Overview(ctx, Query{Subject: "user", Timeframe: Timeframe24H})+ root.End()+ if err == nil {+ t.Fatal("expected source error")+ }++ span := findEndedSpan(recorder.Ended(), "usage.events.query")+ if span == nil {+ t.Fatal("expected usage.events.query span")+ }+ if span.Status().Code != codes.Error {+ t.Fatalf("span status = %v, want error", span.Status().Code)+ }+ for _, event := range span.Events() {+ for _, attr := range event.Attributes {+ if strings.Contains(fmt.Sprint(attr.Value.AsInterface()), "[email protected]") {+ t.Fatalf("span exported source error through %q", attr.Key)+ }+ }+ }+}