Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(metering): tolerate metric label churn (#7156)
r33drichardse3bcfd65cda3
diff --git a/libs/fleet/backend/cmd/usage-meter/main.go b/libs/fleet/backend/cmd/usage-meter/main.goindex 6fcc06e208..66b7b4fc9f 100644--- a/libs/fleet/backend/cmd/usage-meter/main.go+++ b/libs/fleet/backend/cmd/usage-meter/main.go@@ -89,12 +89,14 @@ func run(ctx context.Context, now func() time.Time) error {facts, _ := meter.Int64Counter("cyclops.usage_meter.facts")facts.Add(ctx, int64(result.Inserted), attributeMetric("outcome", "inserted"))facts.Add(ctx, int64(result.Unchanged), attributeMetric("outcome", "unchanged"))+ facts.Add(ctx, int64(result.Unattributed), attributeMetric("outcome", "unattributed"))coverage, _ := meter.Float64Histogram("cyclops.usage_meter.coverage", meteringMetricOptions("KSM source coverage for a materialized hour.")...)coverage.Record(ctx, result.Coverage.Seconds())span.SetAttributes(attribute.Int("meter.sandboxes_discovered", result.Discovered),attribute.Int("meter.facts_inserted", result.Inserted),attribute.Int("meter.facts_unchanged", result.Unchanged),+ attribute.Int("meter.sandboxes_unattributed", result.Unattributed),attribute.Int64("meter.coverage_seconds", int64(result.Coverage.Seconds())),)slog.Info("hourly reservation metering complete",@@ -102,6 +104,7 @@ func run(ctx context.Context, now func() time.Time) error {"discovered", result.Discovered,"inserted", result.Inserted,"unchanged", result.Unchanged,+ "unattributed", result.Unattributed,"coverage_seconds", int64(result.Coverage.Seconds()))return nil}diff --git a/libs/fleet/backend/metering/collector.go b/libs/fleet/backend/metering/collector.goindex de09398974..8fe3cb653e 100644--- a/libs/fleet/backend/metering/collector.go+++ b/libs/fleet/backend/metering/collector.go@@ -26,7 +26,7 @@ type RangeQuerier interface {}type TenantResolver interface {- ResolveTenant(context.Context, string, string, string) (string, error)+ ResolveTenant(context.Context, string, string, string) (string, bool, error)}type FactWriter interface {@@ -79,10 +79,11 @@ type Collector struct {}type CollectResult struct {- Discovered int- Inserted int- Unchanged int- Coverage time.Duration+ Discovered int+ Inserted int+ Unchanged int+ Unattributed int+ Coverage time.Duration}type sandboxIdentity struct {@@ -172,10 +173,14 @@ func (c Collector) CollectHour(ctx context.Context, hourStart time.Time) (Collecif err != nil {return CollectResult{}, fmt.Errorf("integrate readiness for sandbox %s: %w", sandbox.identity.SandboxUID, err)}- tenant, err := c.Tenants.ResolveTenant(ctx, c.ClusterID, sandbox.identity.Namespace, sandbox.identity.SandboxUID)+ tenant, attributed, err := c.Tenants.ResolveTenant(ctx, c.ClusterID, sandbox.identity.Namespace, sandbox.identity.SandboxUID)if err != nil {return CollectResult{}, fmt.Errorf("resolve tenant for sandbox %s: %w", sandbox.identity.SandboxUID, err)}+ if !attributed {+ result.Unattributed+++ continue+ }fact := HourFact{FactID: uuid.New(),LogicalKey: c.ClusterID + "/" + sandbox.identity.SandboxUID + "/" + hourStart.Format(time.RFC3339),@@ -242,22 +247,20 @@ func mergeSandboxSeries(metrics map[string][]Series) (map[string]*sandboxSamples} else if sandbox.identity != identity {return nil, fmt.Errorf("sandbox %s metric labels disagree", identity.SandboxUID)}+ var merged []Sampleswitch metric {case CPUQuery:- if sandbox.cpu != nil {- return nil, fmt.Errorf("sandbox %s has duplicate CPU series", identity.SandboxUID)- }- sandbox.cpu = series.Samples+ merged, err = mergeMetricSamples(sandbox.cpu, series.Samples)+ sandbox.cpu = mergedcase MemoryQuery:- if sandbox.memory != nil {- return nil, fmt.Errorf("sandbox %s has duplicate memory series", identity.SandboxUID)- }- sandbox.memory = series.Samples+ merged, err = mergeMetricSamples(sandbox.memory, series.Samples)+ sandbox.memory = mergedcase ReadyQuery:- if sandbox.ready != nil {- return nil, fmt.Errorf("sandbox %s has duplicate ready series", identity.SandboxUID)- }- sandbox.ready = series.Samples+ merged, err = mergeMetricSamples(sandbox.ready, series.Samples)+ sandbox.ready = merged+ }+ if err != nil {+ return nil, fmt.Errorf("sandbox %s has conflicting %s series: %w", identity.SandboxUID, metric, err)}}}@@ -269,6 +272,22 @@ func mergeSandboxSeries(metrics map[string][]Series) (map[string]*sandboxSamplesreturn out, nil}+func mergeMetricSamples(existing, incoming []Sample) ([]Sample, error) {+ merged := append(append([]Sample(nil), existing...), incoming...)+ sort.Slice(merged, func(i, j int) bool { return merged[i].Timestamp.Before(merged[j].Timestamp) })+ compacted := merged[:0]+ for _, sample := range merged {+ if len(compacted) == 0 || !compacted[len(compacted)-1].Timestamp.Equal(sample.Timestamp) {+ compacted = append(compacted, sample)+ continue+ }+ if compacted[len(compacted)-1].Value != sample.Value {+ return nil, fmt.Errorf("timestamp %s has values %g and %g", sample.Timestamp.Format(time.RFC3339Nano), compacted[len(compacted)-1].Value, sample.Value)+ }+ }+ return compacted, nil+}+func identityFromLabels(labels map[string]string) (sandboxIdentity, error) {poolName := strings.TrimSpace(labels["pool"])if poolName == "" {diff --git a/libs/fleet/backend/metering/collector_test.go b/libs/fleet/backend/metering/collector_test.goindex 2c835a7100..fd498232d3 100644--- a/libs/fleet/backend/metering/collector_test.go+++ b/libs/fleet/backend/metering/collector_test.go@@ -15,8 +15,8 @@ func (f fakeQuerier) QueryRange(_ context.Context, query string, _, _ time.Time,type fakeTenantResolver struct{ tenant string }-func (f fakeTenantResolver) ResolveTenant(context.Context, string, string, string) (string, error) {- return f.tenant, nil+func (f fakeTenantResolver) ResolveTenant(context.Context, string, string, string) (string, bool, error) {+ return f.tenant, f.tenant != "", nil}type fakeFactWriter struct{ facts []HourFact }@@ -71,6 +71,32 @@ func TestCollectorMaterializesReservationHour(t *testing.T) {}}+func TestCollectorSkipsUnattributedSandbox(t *testing.T) {+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)+ labels := map[string]string{+ "namespace": "legacy-system", "sandbox": "sandbox-a", "sandbox_uid": "uid-a",+ "pool": "pool-a", "runtime": "macos",+ }+ writer := &fakeFactWriter{}+ collector := Collector{+ Prometheus: fakeQuerier{+ SourceQuery: {{Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, 1)}},+ CPUQuery: {{Labels: labels, Samples: constantSamples(start, start.Add(time.Hour), 15*time.Second, 4)}},+ MemoryQuery: {{Labels: labels, Samples: constantSamples(start, start.Add(time.Hour), 15*time.Second, float64(4<<30))}},+ },+ Tenants: fakeTenantResolver{}, Writer: writer, ClusterID: "kopf-k3s",+ Step: 15 * time.Second, MaxSampleValidity: time.Minute, MinimumCoverage: 3570 * time.Second,+ }++ result, err := collector.CollectHour(context.Background(), start)+ if err != nil {+ t.Fatal(err)+ }+ if result.Discovered != 1 || result.Unattributed != 1 || result.Inserted != 0 || len(writer.facts) != 0 {+ t.Fatalf("result = %+v, facts = %d", result, len(writer.facts))+ }+}+func TestCollectorFailsClosedOnSourceCoverage(t *testing.T) {start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)collector := Collector{@@ -88,6 +114,44 @@ func TestCollectorFailsClosedOnSourceCoverage(t *testing.T) {}}+func TestMergeSandboxSeriesStitchesLabelChurn(t *testing.T) {+ start := time.Date(2026, 8, 22, 10, 0, 0, 0, time.UTC)+ labels := map[string]string{+ "namespace": "tenant-a", "sandbox": "sandbox-a", "sandbox_uid": "uid-a",+ "pool": "pool-a", "runtime": "kubevirt",+ }+ labelsWithWarmPool := map[string]string{+ "namespace": "tenant-a", "sandbox": "sandbox-a", "sandbox_uid": "uid-a",+ "pool": "pool-a", "warmpool": "pool-a", "runtime": "kubevirt",+ }+ metrics := map[string][]Series{+ CPUQuery: {+ {Labels: labelsWithWarmPool, Samples: []Sample{{Timestamp: start, Value: 4}}},+ {Labels: labels, Samples: []Sample{{Timestamp: start.Add(15 * time.Second), Value: 4}}},+ },+ MemoryQuery: {{Labels: labels, Samples: []Sample{{Timestamp: start, Value: float64(4 << 30)}}}},+ }++ sandboxes, err := mergeSandboxSeries(metrics)+ if err != nil {+ t.Fatal(err)+ }+ if got := sandboxes["tenant-a/uid-a"].cpu; len(got) != 2 || !got[0].Timestamp.Equal(start) || !got[1].Timestamp.Equal(start.Add(15*time.Second)) {+ t.Fatalf("merged CPU samples = %+v", got)+ }+}++func TestMergeMetricSamplesRejectsConflictingTimestamp(t *testing.T) {+ timestamp := time.Date(2026, 8, 22, 10, 0, 0, 0, time.UTC)+ _, err := mergeMetricSamples(+ []Sample{{Timestamp: timestamp, Value: 4}},+ []Sample{{Timestamp: timestamp, Value: 8}},+ )+ if err == nil || !strings.Contains(err.Error(), "has values 4 and 8") {+ t.Fatalf("error = %v, want conflicting values", err)+ }+}+func TestIdentityFromLabelsFallsBackToWarmPoolLabel(t *testing.T) {identity, err := identityFromLabels(map[string]string{"namespace": "tenant-a",diff --git a/libs/fleet/backend/metering/postgres.go b/libs/fleet/backend/metering/postgres.goindex 31735fe4a0..5e7ae59565 100644--- a/libs/fleet/backend/metering/postgres.go+++ b/libs/fleet/backend/metering/postgres.go@@ -2,12 +2,14 @@ package meteringimport ("context"+ "errors""fmt""strings""time""github.com/google/uuid""github.com/jackc/pgx/v5"+ "github.com/jackc/pgx/v5/pgconn""github.com/jackc/pgx/v5/pgxpool")@@ -55,15 +57,19 @@ func (s *PostgresStore) Close() {}}-func (s *PostgresStore) ResolveTenant(ctx context.Context, clusterID, namespace, sandboxUID string) (string, error) {+func (s *PostgresStore) ResolveTenant(ctx context.Context, clusterID, namespace, sandboxUID string) (string, bool, error) {var tenant stringif err := s.pool.QueryRow(ctx, `select k8s_api.sandbox_meter_tenant($1, $2, $3)`, clusterID, namespace, sandboxUID).Scan(&tenant); err != nil {- return "", fmt.Errorf("query sandbox tenant: %w", err)+ var postgresError *pgconn.PgError+ if errors.As(err, &postgresError) && postgresError.Code == "P0001" && postgresError.Message == "sandbox tenant identity is unavailable" {+ return "", false, nil+ }+ return "", false, fmt.Errorf("query sandbox tenant: %w", err)}if strings.TrimSpace(tenant) == "" {- return "", fmt.Errorf("sandbox tenant is empty")+ return "", false, fmt.Errorf("sandbox tenant is empty")}- return tenant, nil+ return tenant, true, nil}func (s *PostgresStore) AppendFact(ctx context.Context, fact HourFact) (bool, error) {