Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(metering): tolerate metric label churn (#7156)

r33drichardse3bcfd65cda3
diff --git a/libs/fleet/backend/cmd/usage-meter/main.go b/libs/fleet/backend/cmd/usage-meter/main.go
index 6fcc06e208..66b7b4fc9f 100644
--- a/libs/fleet/backend/cmd/usage-meter/main.go
+++ b/libs/fleet/backend/cmd/usage-meter/main.go
@@ -89,12 +89,14 @@ func run(ctx context.Context, now func() time.Time) error {
facts, _ := meter.Int64Counter("cyclops.usage_meter.facts")
facts.Add(ctx, int64(result.Inserted), attributeMetric("outcome", "inserted"))
facts.Add(ctx, int64(result.Unchanged), attributeMetric("outcome", "unchanged"))
+ facts.Add(ctx, int64(result.Unattributed), attributeMetric("outcome", "unattributed"))
coverage, _ := meter.Float64Histogram("cyclops.usage_meter.coverage", meteringMetricOptions("KSM source coverage for a materialized hour.")...)
coverage.Record(ctx, result.Coverage.Seconds())
span.SetAttributes(
attribute.Int("meter.sandboxes_discovered", result.Discovered),
attribute.Int("meter.facts_inserted", result.Inserted),
attribute.Int("meter.facts_unchanged", result.Unchanged),
+ attribute.Int("meter.sandboxes_unattributed", result.Unattributed),
attribute.Int64("meter.coverage_seconds", int64(result.Coverage.Seconds())),
)
slog.Info("hourly reservation metering complete",
@@ -102,6 +104,7 @@ func run(ctx context.Context, now func() time.Time) error {
"discovered", result.Discovered,
"inserted", result.Inserted,
"unchanged", result.Unchanged,
+ "unattributed", result.Unattributed,
"coverage_seconds", int64(result.Coverage.Seconds()))
return nil
}
diff --git a/libs/fleet/backend/metering/collector.go b/libs/fleet/backend/metering/collector.go
index de09398974..8fe3cb653e 100644
--- a/libs/fleet/backend/metering/collector.go
+++ b/libs/fleet/backend/metering/collector.go
@@ -26,7 +26,7 @@ type RangeQuerier interface {
}
type TenantResolver interface {
- ResolveTenant(context.Context, string, string, string) (string, error)
+ ResolveTenant(context.Context, string, string, string) (string, bool, error)
}
type FactWriter interface {
@@ -79,10 +79,11 @@ type Collector struct {
}
type CollectResult struct {
- Discovered int
- Inserted int
- Unchanged int
- Coverage time.Duration
+ Discovered int
+ Inserted int
+ Unchanged int
+ Unattributed int
+ Coverage time.Duration
}
type sandboxIdentity struct {
@@ -172,10 +173,14 @@ func (c Collector) CollectHour(ctx context.Context, hourStart time.Time) (Collec
if err != nil {
return CollectResult{}, fmt.Errorf("integrate readiness for sandbox %s: %w", sandbox.identity.SandboxUID, err)
}
- tenant, err := c.Tenants.ResolveTenant(ctx, c.ClusterID, sandbox.identity.Namespace, sandbox.identity.SandboxUID)
+ tenant, attributed, err := c.Tenants.ResolveTenant(ctx, c.ClusterID, sandbox.identity.Namespace, sandbox.identity.SandboxUID)
if err != nil {
return CollectResult{}, fmt.Errorf("resolve tenant for sandbox %s: %w", sandbox.identity.SandboxUID, err)
}
+ if !attributed {
+ result.Unattributed++
+ continue
+ }
fact := HourFact{
FactID: uuid.New(),
LogicalKey: c.ClusterID + "/" + sandbox.identity.SandboxUID + "/" + hourStart.Format(time.RFC3339),
@@ -242,22 +247,20 @@ func mergeSandboxSeries(metrics map[string][]Series) (map[string]*sandboxSamples
} else if sandbox.identity != identity {
return nil, fmt.Errorf("sandbox %s metric labels disagree", identity.SandboxUID)
}
+ var merged []Sample
switch metric {
case CPUQuery:
- if sandbox.cpu != nil {
- return nil, fmt.Errorf("sandbox %s has duplicate CPU series", identity.SandboxUID)
- }
- sandbox.cpu = series.Samples
+ merged, err = mergeMetricSamples(sandbox.cpu, series.Samples)
+ sandbox.cpu = merged
case MemoryQuery:
- if sandbox.memory != nil {
- return nil, fmt.Errorf("sandbox %s has duplicate memory series", identity.SandboxUID)
- }
- sandbox.memory = series.Samples
+ merged, err = mergeMetricSamples(sandbox.memory, series.Samples)
+ sandbox.memory = merged
case ReadyQuery:
- if sandbox.ready != nil {
- return nil, fmt.Errorf("sandbox %s has duplicate ready series", identity.SandboxUID)
- }
- sandbox.ready = series.Samples
+ merged, err = mergeMetricSamples(sandbox.ready, series.Samples)
+ sandbox.ready = merged
+ }
+ if err != nil {
+ return nil, fmt.Errorf("sandbox %s has conflicting %s series: %w", identity.SandboxUID, metric, err)
}
}
}
@@ -269,6 +272,22 @@ func mergeSandboxSeries(metrics map[string][]Series) (map[string]*sandboxSamples
return out, nil
}
+func mergeMetricSamples(existing, incoming []Sample) ([]Sample, error) {
+ merged := append(append([]Sample(nil), existing...), incoming...)
+ sort.Slice(merged, func(i, j int) bool { return merged[i].Timestamp.Before(merged[j].Timestamp) })
+ compacted := merged[:0]
+ for _, sample := range merged {
+ if len(compacted) == 0 || !compacted[len(compacted)-1].Timestamp.Equal(sample.Timestamp) {
+ compacted = append(compacted, sample)
+ continue
+ }
+ if compacted[len(compacted)-1].Value != sample.Value {
+ return nil, fmt.Errorf("timestamp %s has values %g and %g", sample.Timestamp.Format(time.RFC3339Nano), compacted[len(compacted)-1].Value, sample.Value)
+ }
+ }
+ return compacted, nil
+}
+
func identityFromLabels(labels map[string]string) (sandboxIdentity, error) {
poolName := strings.TrimSpace(labels["pool"])
if poolName == "" {
diff --git a/libs/fleet/backend/metering/collector_test.go b/libs/fleet/backend/metering/collector_test.go
index 2c835a7100..fd498232d3 100644
--- a/libs/fleet/backend/metering/collector_test.go
+++ b/libs/fleet/backend/metering/collector_test.go
@@ -15,8 +15,8 @@ func (f fakeQuerier) QueryRange(_ context.Context, query string, _, _ time.Time,
type fakeTenantResolver struct{ tenant string }
-func (f fakeTenantResolver) ResolveTenant(context.Context, string, string, string) (string, error) {
- return f.tenant, nil
+func (f fakeTenantResolver) ResolveTenant(context.Context, string, string, string) (string, bool, error) {
+ return f.tenant, f.tenant != "", nil
}
type fakeFactWriter struct{ facts []HourFact }
@@ -71,6 +71,32 @@ func TestCollectorMaterializesReservationHour(t *testing.T) {
}
}
+func TestCollectorSkipsUnattributedSandbox(t *testing.T) {
+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)
+ labels := map[string]string{
+ "namespace": "legacy-system", "sandbox": "sandbox-a", "sandbox_uid": "uid-a",
+ "pool": "pool-a", "runtime": "macos",
+ }
+ writer := &fakeFactWriter{}
+ collector := Collector{
+ Prometheus: fakeQuerier{
+ SourceQuery: {{Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, 1)}},
+ CPUQuery: {{Labels: labels, Samples: constantSamples(start, start.Add(time.Hour), 15*time.Second, 4)}},
+ MemoryQuery: {{Labels: labels, Samples: constantSamples(start, start.Add(time.Hour), 15*time.Second, float64(4<<30))}},
+ },
+ Tenants: fakeTenantResolver{}, Writer: writer, ClusterID: "kopf-k3s",
+ Step: 15 * time.Second, MaxSampleValidity: time.Minute, MinimumCoverage: 3570 * time.Second,
+ }
+
+ result, err := collector.CollectHour(context.Background(), start)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.Discovered != 1 || result.Unattributed != 1 || result.Inserted != 0 || len(writer.facts) != 0 {
+ t.Fatalf("result = %+v, facts = %d", result, len(writer.facts))
+ }
+}
+
func TestCollectorFailsClosedOnSourceCoverage(t *testing.T) {
start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)
collector := Collector{
@@ -88,6 +114,44 @@ func TestCollectorFailsClosedOnSourceCoverage(t *testing.T) {
}
}
+func TestMergeSandboxSeriesStitchesLabelChurn(t *testing.T) {
+ start := time.Date(2026, 8, 22, 10, 0, 0, 0, time.UTC)
+ labels := map[string]string{
+ "namespace": "tenant-a", "sandbox": "sandbox-a", "sandbox_uid": "uid-a",
+ "pool": "pool-a", "runtime": "kubevirt",
+ }
+ labelsWithWarmPool := map[string]string{
+ "namespace": "tenant-a", "sandbox": "sandbox-a", "sandbox_uid": "uid-a",
+ "pool": "pool-a", "warmpool": "pool-a", "runtime": "kubevirt",
+ }
+ metrics := map[string][]Series{
+ CPUQuery: {
+ {Labels: labelsWithWarmPool, Samples: []Sample{{Timestamp: start, Value: 4}}},
+ {Labels: labels, Samples: []Sample{{Timestamp: start.Add(15 * time.Second), Value: 4}}},
+ },
+ MemoryQuery: {{Labels: labels, Samples: []Sample{{Timestamp: start, Value: float64(4 << 30)}}}},
+ }
+
+ sandboxes, err := mergeSandboxSeries(metrics)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got := sandboxes["tenant-a/uid-a"].cpu; len(got) != 2 || !got[0].Timestamp.Equal(start) || !got[1].Timestamp.Equal(start.Add(15*time.Second)) {
+ t.Fatalf("merged CPU samples = %+v", got)
+ }
+}
+
+func TestMergeMetricSamplesRejectsConflictingTimestamp(t *testing.T) {
+ timestamp := time.Date(2026, 8, 22, 10, 0, 0, 0, time.UTC)
+ _, err := mergeMetricSamples(
+ []Sample{{Timestamp: timestamp, Value: 4}},
+ []Sample{{Timestamp: timestamp, Value: 8}},
+ )
+ if err == nil || !strings.Contains(err.Error(), "has values 4 and 8") {
+ t.Fatalf("error = %v, want conflicting values", err)
+ }
+}
+
func TestIdentityFromLabelsFallsBackToWarmPoolLabel(t *testing.T) {
identity, err := identityFromLabels(map[string]string{
"namespace": "tenant-a",
diff --git a/libs/fleet/backend/metering/postgres.go b/libs/fleet/backend/metering/postgres.go
index 31735fe4a0..5e7ae59565 100644
--- a/libs/fleet/backend/metering/postgres.go
+++ b/libs/fleet/backend/metering/postgres.go
@@ -2,12 +2,14 @@ package metering
import (
"context"
+ "errors"
"fmt"
"strings"
"time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgconn"
"github.com/jackc/pgx/v5/pgxpool"
)
@@ -55,15 +57,19 @@ func (s *PostgresStore) Close() {
}
}
-func (s *PostgresStore) ResolveTenant(ctx context.Context, clusterID, namespace, sandboxUID string) (string, error) {
+func (s *PostgresStore) ResolveTenant(ctx context.Context, clusterID, namespace, sandboxUID string) (string, bool, error) {
var tenant string
if err := s.pool.QueryRow(ctx, `select k8s_api.sandbox_meter_tenant($1, $2, $3)`, clusterID, namespace, sandboxUID).Scan(&tenant); err != nil {
- return "", fmt.Errorf("query sandbox tenant: %w", err)
+ var postgresError *pgconn.PgError
+ if errors.As(err, &postgresError) && postgresError.Code == "P0001" && postgresError.Message == "sandbox tenant identity is unavailable" {
+ return "", false, nil
+ }
+ return "", false, fmt.Errorf("query sandbox tenant: %w", err)
}
if strings.TrimSpace(tenant) == "" {
- return "", fmt.Errorf("sandbox tenant is empty")
+ return "", false, fmt.Errorf("sandbox tenant is empty")
}
- return tenant, nil
+ return tenant, true, nil
}
func (s *PostgresStore) AppendFact(ctx context.Context, fact HourFact) (bool, error) {