Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cyclops): trace usage latency breakdown (#7118)

r33drichards3d797367152b
diff --git a/libs/fleet/backend/handlers/usage.go b/libs/fleet/backend/handlers/usage.go
index 0613fbfc93..d4a9155a3e 100644
--- a/libs/fleet/backend/handlers/usage.go
+++ b/libs/fleet/backend/handlers/usage.go
@@ -8,6 +8,9 @@ import (
"strings"
"cyclops-cs-backend/usage"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/codes"
+ "go.opentelemetry.io/otel/trace"
)
type UsageTimeframe = usage.Timeframe
@@ -45,6 +48,9 @@ func parseUsageTimeframe(raw string) (UsageTimeframe, UsageInterval, bool) {
}
func (h Handlers) authorizeUsage(w http.ResponseWriter, r *http.Request) (UsageQuery, UsageInterval, bool) {
+ ctx, span := handlerTracer().Start(r.Context(), "usage.authorize")
+ defer span.End()
+
u := currentUser(r)
if u == nil || u.ID == "" {
writeErr(w, 401, "authentication required")
@@ -55,7 +61,7 @@ func (h Handlers) authorizeUsage(w http.ResponseWriter, r *http.Request) (UsageQ
writeErr(w, 400, "timeframe must be one of 24h, 7d, or 30d")
return UsageQuery{}, "", false
}
- enabled, err := h.usageEnabled(r.Context(), u)
+ enabled, err := h.usageEnabled(ctx, u)
if err != nil || !enabled {
if err != nil {
slog.WarnContext(r.Context(), "usage access evaluation failed", "err", err)
@@ -63,7 +69,7 @@ func (h Handlers) authorizeUsage(w http.ResponseWriter, r *http.Request) (UsageQ
writeErr(w, 403, "usage preview is not enabled")
return UsageQuery{}, "", false
}
- admin, err := h.isAdmin(r.Context(), u)
+ admin, err := h.isAdmin(ctx, u)
if err != nil {
admin = false
}
@@ -78,10 +84,20 @@ func (h Handlers) authorizeUsage(w http.ResponseWriter, r *http.Request) (UsageQ
writeErr(w, 400, "subject is too long")
return UsageQuery{}, "", false
}
+ span.SetAttributes(
+ attribute.String("usage.timeframe", string(tf)),
+ attribute.String("usage.interval", string(iv)),
+ attribute.Bool("usage.admin", admin),
+ attribute.Bool("usage.subject_override", sub != u.ID),
+ attribute.Bool("usage.authorized", true),
+ )
return UsageQuery{ActorSubject: u.ID, Subject: sub, Admin: admin, Timeframe: tf}, iv, true
}
func (h Handlers) GetUsageOverview(w http.ResponseWriter, r *http.Request) {
+ ctx, span := handlerTracer().Start(r.Context(), "usage.overview")
+ defer span.End()
+ r = r.WithContext(ctx)
w.Header().Set("Cache-Control", "private, no-store")
q, _, ok := h.authorizeUsage(w, r)
if !ok {
@@ -91,8 +107,10 @@ func (h Handlers) GetUsageOverview(w http.ResponseWriter, r *http.Request) {
writeErr(w, 503, "usage data provider is not configured")
return
}
+ span.SetAttributes(attribute.String("usage.timeframe", string(q.Timeframe)))
v, err := h.Usage.Overview(r.Context(), q)
if err != nil {
+ markUsageHandlerError(span, "usage overview failed")
writeErr(w, 502, "usage data is temporarily unavailable")
return
}
@@ -100,6 +118,9 @@ func (h Handlers) GetUsageOverview(w http.ResponseWriter, r *http.Request) {
}
func (h Handlers) GetUsagePoolDetail(w http.ResponseWriter, r *http.Request) {
+ ctx, span := handlerTracer().Start(r.Context(), "usage.pool_detail")
+ defer span.End()
+ r = r.WithContext(ctx)
w.Header().Set("Cache-Control", "private, no-store")
q, iv, ok := h.authorizeUsage(w, r)
if !ok {
@@ -114,14 +135,23 @@ func (h Handlers) GetUsagePoolDetail(w http.ResponseWriter, r *http.Request) {
writeErr(w, 503, "usage data provider is not configured")
return
}
+ span.SetAttributes(
+ attribute.String("usage.timeframe", string(q.Timeframe)),
+ attribute.String("usage.interval", string(iv)),
+ )
v, err := h.Usage.PoolDetail(r.Context(), UsagePoolQuery{Query: q, PoolID: pool, Interval: iv})
if errors.Is(err, usage.ErrPoolNotFound) {
writeErr(w, 404, "usage pool was not found")
return
}
if err != nil {
+ markUsageHandlerError(span, "usage pool detail failed")
writeErr(w, 502, "usage data is temporarily unavailable")
return
}
writeJSON(w, 200, v)
}
+
+func markUsageHandlerError(span trace.Span, description string) {
+ span.SetStatus(codes.Error, description)
+}
diff --git a/libs/fleet/backend/handlers/usage_test.go b/libs/fleet/backend/handlers/usage_test.go
index fdf1306222..0802d3f9e5 100644
--- a/libs/fleet/backend/handlers/usage_test.go
+++ b/libs/fleet/backend/handlers/usage_test.go
@@ -8,6 +8,10 @@ import (
"net/http"
"net/http/httptest"
"testing"
+
+ "go.opentelemetry.io/otel"
+ sdktrace "go.opentelemetry.io/otel/sdk/trace"
+ "go.opentelemetry.io/otel/sdk/trace/tracetest"
)
type fakeUsageProvider struct {
@@ -105,3 +109,46 @@ func TestUsageProviderErrorsAreRedacted(t *testing.T) {
}
})
}
+
+func TestUsagePoolDetailCreatesHandlerAndAuthorizationSpans(t *testing.T) {
+ recorder := tracetest.NewSpanRecorder()
+ provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
+ previousProvider := otel.GetTracerProvider()
+ otel.SetTracerProvider(provider)
+ t.Cleanup(func() { otel.SetTracerProvider(previousProvider) })
+
+ ctx, root := provider.Tracer("test").Start(context.Background(), "request")
+ request := withUser(
+ httptest.NewRequest(http.MethodGet, "/api/usage/pool?timeframe=24h&pool=pool-should-not-appear", nil).WithContext(ctx),
+ &auth.User{ID: "user-should-not-appear"},
+ )
+ response := httptest.NewRecorder()
+
+ uh(&fakeUsageProvider{}, true, false).GetUsagePoolDetail(response, request)
+ root.End()
+
+ if response.Code != http.StatusOK {
+ t.Fatalf("status = %d, want %d", response.Code, http.StatusOK)
+ }
+ for _, name := range []string{"usage.pool_detail", "usage.authorize"} {
+ if handlerSpanByName(recorder.Ended(), name) == nil {
+ t.Fatalf("expected %q span", name)
+ }
+ }
+ for _, span := range recorder.Ended() {
+ for _, attr := range span.Attributes() {
+ if attr.Value.AsString() == "user-should-not-appear" || attr.Value.AsString() == "pool-should-not-appear" {
+ t.Fatalf("span %q contains sensitive/high-cardinality attribute %q", span.Name(), attr.Key)
+ }
+ }
+ }
+}
+
+func handlerSpanByName(spans []sdktrace.ReadOnlySpan, name string) sdktrace.ReadOnlySpan {
+ for _, span := range spans {
+ if span.Name() == name {
+ return span
+ }
+ }
+ return nil
+}
diff --git a/libs/fleet/backend/usage/provider.go b/libs/fleet/backend/usage/provider.go
index 69e87ded38..d48333d636 100644
--- a/libs/fleet/backend/usage/provider.go
+++ b/libs/fleet/backend/usage/provider.go
@@ -9,6 +9,8 @@ import (
"time"
"cyclops-cs-backend/identity"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/trace"
)
var ErrPoolNotFound = errors.New("usage pool not found")
@@ -65,12 +67,17 @@ func (p *provider) PoolDetail(ctx context.Context, query PoolQuery) (PoolDetailR
if !ok {
return PoolDetailResponse{}, fmt.Errorf("%w: %s", ErrPoolNotFound, query.PoolID)
}
+ _, bucketSpan := usageTracer().Start(ctx, "usage.buckets.build", trace.WithAttributes(
+ attribute.String("usage.interval", string(query.Interval)),
+ ))
response := PoolDetailResponse{DataAsOf: data.asOf, Partial: data.partial, Pool: PoolIdentity{ID: pool.id, Name: pool.name}, Buckets: make([]Bucket, 0)}
for start := data.start; start.Before(data.end); start = start.Add(step) {
key := bucketKey{namespace: namespace, pool: poolName, start: start}
totals := data.buckets[key]
response.Buckets = append(response.Buckets, Bucket{Start: start, End: start.Add(step), CPUConsumed: totals.cpuConsumed, CPUProvisioned: totals.cpuProvisioned, MemoryConsumed: totals.memoryConsumed, MemoryProvisioned: totals.memoryProvisioned})
}
+ bucketSpan.SetAttributes(attribute.Int("usage.bucket_count", len(response.Buckets)))
+ bucketSpan.End()
return response, nil
}
@@ -99,26 +106,74 @@ type bucketKey struct {
}
func (p *provider) load(ctx context.Context, query Query) (usageData, error) {
+ ctx, span := usageTracer().Start(ctx, "usage.load", trace.WithAttributes(
+ attribute.String("usage.timeframe", string(query.Timeframe)),
+ attribute.Bool("usage.admin", query.Admin),
+ attribute.Bool("usage.subject_override", query.ActorSubject != "" && query.ActorSubject != query.Subject),
+ ))
+ defer span.End()
+
if p.events == nil || p.allocations == nil {
- return usageData{}, fmt.Errorf("usage provider is not configured")
+ err := fmt.Errorf("usage provider is not configured")
+ markUsageSpanError(span, "usage provider unavailable")
+ return usageData{}, err
}
start, end, step, err := usageWindow(query.Timeframe, p.clock())
if err != nil {
+ markUsageSpanError(span, "invalid usage window")
return usageData{}, err
}
- events, err := p.events.Events(ctx, identity.PersonalGroup(ctx, query.Subject), start, end)
+ span.SetAttributes(attribute.Int64("usage.window_seconds", int64(end.Sub(start).Seconds())))
+
+ eventsCtx, eventsSpan := usageTracer().Start(ctx, "usage.events.query")
+ events, err := p.events.Events(eventsCtx, identity.PersonalGroup(eventsCtx, query.Subject), start, end)
if err != nil {
+ markUsageSpanError(eventsSpan, "usage event query failed")
+ eventsSpan.End()
+ markUsageSpanError(span, "usage event query failed")
return usageData{}, fmt.Errorf("read usage events: %w", err)
}
+ eventsSpan.SetAttributes(attribute.Int("usage.event_count", len(events)))
+ eventsSpan.End()
+
+ _, segmentsSpan := usageTracer().Start(ctx, "usage.segments.build", trace.WithAttributes(
+ attribute.Int("usage.event_count", len(events)),
+ ))
segments, namespaces, partial := buildSegments(events, start, end)
+ segmentsSpan.SetAttributes(
+ attribute.Int("usage.segment_count", len(segments)),
+ attribute.Int("usage.namespace_count", len(namespaces)),
+ attribute.Bool("usage.partial", partial),
+ )
+ segmentsSpan.End()
+
data := usageData{start: start, end: end, asOf: end, partial: partial, pools: make([]usageTotals, 0), buckets: make(map[bucketKey]usageTotals)}
if len(segments) == 0 || len(namespaces) == 0 {
+ span.SetAttributes(
+ attribute.Int("usage.pool_count", 0),
+ attribute.Int("usage.bucket_count", 0),
+ attribute.Bool("usage.partial", data.partial),
+ )
return data, nil
}
- allocations, asOf, sourcePartial, err := p.allocations.Allocations(ctx, start, end, step, namespaces)
+
+ allocationsCtx, allocationsSpan := usageTracer().Start(ctx, "usage.allocations.query", trace.WithAttributes(
+ attribute.Int("usage.namespace_count", len(namespaces)),
+ attribute.Int64("usage.step_seconds", int64(step.Seconds())),
+ ))
+ allocations, asOf, sourcePartial, err := p.allocations.Allocations(allocationsCtx, start, end, step, namespaces)
if err != nil {
+ markUsageSpanError(allocationsSpan, "OpenCost allocation query failed")
+ allocationsSpan.End()
+ markUsageSpanError(span, "OpenCost allocation query failed")
return usageData{}, fmt.Errorf("read OpenCost allocations: %w", err)
}
+ allocationsSpan.SetAttributes(
+ attribute.Int("usage.allocation_count", len(allocations)),
+ attribute.Bool("usage.partial", sourcePartial),
+ )
+ allocationsSpan.End()
+
if asOf.IsZero() {
asOf = start
sourcePartial = true
@@ -129,6 +184,10 @@ func (p *provider) load(ctx context.Context, query Query) (usageData, error) {
sourcePartial = true
}
data.partial = data.partial || sourcePartial || data.asOf.Before(end)
+
+ _, attributionSpan := usageTracer().Start(ctx, "usage.allocations.attribute", trace.WithAttributes(
+ attribute.Int("usage.allocation_count", len(allocations)),
+ ))
poolTotals := make(map[string]usageTotals)
for _, allocation := range allocations {
matched, allocationPartial := attributeAllocation(allocation, segments, start, end, step, poolTotals, data.buckets)
@@ -143,6 +202,17 @@ func (p *provider) load(ctx context.Context, query Query) (usageData, error) {
}
disambiguatePoolNames(data.pools)
sort.Slice(data.pools, func(i, j int) bool { return data.pools[i].id < data.pools[j].id })
+ attributionSpan.SetAttributes(
+ attribute.Int("usage.pool_count", len(data.pools)),
+ attribute.Int("usage.bucket_count", len(data.buckets)),
+ attribute.Bool("usage.partial", data.partial),
+ )
+ attributionSpan.End()
+ span.SetAttributes(
+ attribute.Int("usage.pool_count", len(data.pools)),
+ attribute.Int("usage.bucket_count", len(data.buckets)),
+ attribute.Bool("usage.partial", data.partial),
+ )
return data, nil
}
diff --git a/libs/fleet/backend/usage/tracing.go b/libs/fleet/backend/usage/tracing.go
new file mode 100644
index 0000000000..7ff0715284
--- /dev/null
+++ b/libs/fleet/backend/usage/tracing.go
@@ -0,0 +1,15 @@
+package usage
+
+import (
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/codes"
+ "go.opentelemetry.io/otel/trace"
+)
+
+func usageTracer() trace.Tracer {
+ return otel.Tracer("cyclops-cs-backend/usage")
+}
+
+func markUsageSpanError(span trace.Span, description string) {
+ span.SetStatus(codes.Error, description)
+}
diff --git a/libs/fleet/backend/usage/tracing_test.go b/libs/fleet/backend/usage/tracing_test.go
new file mode 100644
index 0000000000..8b8662376d
--- /dev/null
+++ b/libs/fleet/backend/usage/tracing_test.go
@@ -0,0 +1,132 @@
+package usage
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/codes"
+ sdktrace "go.opentelemetry.io/otel/sdk/trace"
+ "go.opentelemetry.io/otel/sdk/trace/tracetest"
+)
+
+func TestProviderPoolDetailCreatesLatencyBreakdownSpans(t *testing.T) {
+ cutoff := time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC)
+ recorder := tracetest.NewSpanRecorder()
+ provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
+ previousProvider := otel.GetTracerProvider()
+ otel.SetTracerProvider(provider)
+ t.Cleanup(func() { otel.SetTracerProvider(previousProvider) })
+
+ ctx, root := provider.Tracer("test").Start(context.Background(), "request")
+ usageProvider := NewProvider(
+ &fakeEventStore{events: []SandboxEvent{{
+ EventID: mustUUID("00000000-0000-0000-0000-000000000101"), Namespace: "ns-a", SandboxName: "sandbox-a", SandboxUID: "uid-a",
+ PoolName: "pool-a", Runtime: "kubevirt", VMName: "vm-a", EventType: "Added", ObservedAt: cutoff.Add(-25 * time.Hour),
+ }}},
+ &fakeAllocationClient{asOf: cutoff, allocations: []Allocation{{
+ Start: cutoff.Add(-time.Hour), End: cutoff, Namespace: "ns-a", Pod: "virt-launcher-vm-a-abc",
+ Minutes: 60, CPUUsageAverage: 2, CPURequestAverage: 4,
+ RAMUsageAverageBytes: 3 * gibibyte, RAMRequestAverageBytes: 6 * gibibyte,
+ }}},
+ func() time.Time { return cutoff },
+ )
+
+ _, err := usageProvider.PoolDetail(ctx, PoolQuery{
+ Query: Query{Subject: "user-should-not-appear", Timeframe: Timeframe24H},
+ PoolID: "ns-a:pool-a",
+ Interval: IntervalHour,
+ })
+ root.End()
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ for _, name := range []string{
+ "usage.load",
+ "usage.events.query",
+ "usage.segments.build",
+ "usage.allocations.query",
+ "usage.allocations.attribute",
+ "usage.buckets.build",
+ } {
+ if findEndedSpan(recorder.Ended(), name) == nil {
+ t.Fatalf("expected %q span", name)
+ }
+ }
+
+ loadSpan := findEndedSpan(recorder.Ended(), "usage.load")
+ assertSpanAttribute(t, loadSpan.Attributes(), "usage.timeframe", "24h")
+ assertSpanAttribute(t, findEndedSpan(recorder.Ended(), "usage.events.query").Attributes(), "usage.event_count", int64(1))
+ assertSpanAttribute(t, findEndedSpan(recorder.Ended(), "usage.allocations.query").Attributes(), "usage.allocation_count", int64(1))
+ assertSpanAttribute(t, findEndedSpan(recorder.Ended(), "usage.buckets.build").Attributes(), "usage.bucket_count", int64(24))
+
+ for _, span := range recorder.Ended() {
+ for _, attr := range span.Attributes() {
+ if attr.Value.AsString() == "user-should-not-appear" || attr.Value.AsString() == "ns-a:pool-a" {
+ t.Fatalf("span %q contains sensitive/high-cardinality attribute %q", span.Name(), attr.Key)
+ }
+ }
+ }
+}
+
+func findEndedSpan(spans []sdktrace.ReadOnlySpan, name string) sdktrace.ReadOnlySpan {
+ for _, span := range spans {
+ if span.Name() == name {
+ return span
+ }
+ }
+ return nil
+}
+
+func assertSpanAttribute(t *testing.T, attributes []attribute.KeyValue, key string, want any) {
+ t.Helper()
+ for _, attr := range attributes {
+ if string(attr.Key) == key {
+ if got := attr.Value.AsInterface(); got != want {
+ t.Fatalf("attribute %s = %#v, want %#v", key, got, want)
+ }
+ return
+ }
+ }
+ t.Fatalf("missing attribute %s", key)
+}
+
+func TestProviderErrorSpansDoNotExportSourceMessages(t *testing.T) {
+ recorder := tracetest.NewSpanRecorder()
+ provider := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
+ previousProvider := otel.GetTracerProvider()
+ otel.SetTracerProvider(provider)
+ t.Cleanup(func() { otel.SetTracerProvider(previousProvider) })
+
+ ctx, root := provider.Tracer("test").Start(context.Background(), "request")
+ _, err := NewProvider(
+ &fakeEventStore{err: errors.New("postgres://[email protected]: source unavailable")},
+ &fakeAllocationClient{},
+ func() time.Time { return time.Date(2026, 8, 19, 10, 0, 0, 0, time.UTC) },
+ ).Overview(ctx, Query{Subject: "user", Timeframe: Timeframe24H})
+ root.End()
+ if err == nil {
+ t.Fatal("expected source error")
+ }
+
+ span := findEndedSpan(recorder.Ended(), "usage.events.query")
+ if span == nil {
+ t.Fatal("expected usage.events.query span")
+ }
+ if span.Status().Code != codes.Error {
+ t.Fatalf("span status = %v, want error", span.Status().Code)
+ }
+ for _, event := range span.Events() {
+ for _, attr := range event.Attributes {
+ if strings.Contains(fmt.Sprint(attr.Value.AsInterface()), "[email protected]") {
+ t.Fatalf("span exported source error through %q", attr.Key)
+ }
+ }
+ }
+}