Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(cyclops): materialize hourly sandbox reservations (#7150)
r33drichards268723a1db4e
diff --git a/libs/fleet/backend/Dockerfile b/libs/fleet/backend/Dockerfileindex 7137b23aa0..28b726bdc9 100644--- a/libs/fleet/backend/Dockerfile+++ b/libs/fleet/backend/Dockerfile@@ -10,7 +10,8 @@ RUN go mod tidyRUN go install github.com/swaggo/swag/cmd/[email protected] && \/go/bin/swag init -g main.go -o ./docs --parseDependency --parseInternalRUN CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-cs-backend . \- && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-db-migrate ./cmd/db-migrate+ && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-db-migrate ./cmd/db-migrate \+ && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-usage-meter ./cmd/usage-meterFROM --platform=linux/amd64 rust:1.94-bookworm AS rust-builderWORKDIR /src/k8s-state-query@@ -23,6 +24,7 @@ FROM gcr.io/distroless/cc-debian12:nonroot AS runtimeWORKDIR /appCOPY --from=go-builder /out/cyclops-cs-backend /app/cyclops-cs-backendCOPY --from=go-builder /out/cyclops-db-migrate /app/cyclops-db-migrate+COPY --from=go-builder /out/cyclops-usage-meter /app/cyclops-usage-meterCOPY --from=rust-builder /out-k8s-state-projector /app/k8s-state-projectorUSER nonroot:nonrootEXPOSE 8080diff --git a/libs/fleet/backend/cmd/db-migrate/main.go b/libs/fleet/backend/cmd/db-migrate/main.goindex 8417bc9b48..bb061c3593 100644--- a/libs/fleet/backend/cmd/db-migrate/main.go+++ b/libs/fleet/backend/cmd/db-migrate/main.go@@ -36,6 +36,7 @@ func main() {RoleAdmin: required("STATE_ROLE_ADMIN_DATABASE_URL"),Metabase: required("METABASE_DATABASE_URL"),Usage: required("USAGE_DATABASE_URL"),+ Meter: required("METER_DATABASE_URL"),},}if err := database.Run(context.Background(), config); err != nil {diff --git a/libs/fleet/backend/cmd/usage-meter/main.go b/libs/fleet/backend/cmd/usage-meter/main.gonew file mode 100644index 0000000000..df54aa9b52--- /dev/null+++ b/libs/fleet/backend/cmd/usage-meter/main.go@@ -0,0 +1,176 @@+package main++import (+ "context"+ "fmt"+ "log/slog"+ "os"+ "time"++ "cyclops-cs-backend/metering"+ "cyclops-cs-backend/telemetry"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/codes"+ "go.opentelemetry.io/otel/metric"+)++func main() {+ slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))+ if err := run(context.Background(), time.Now); err != nil {+ slog.Error("hourly reservation metering failed", "error", err)+ os.Exit(1)+ }+}++func run(ctx context.Context, now func() time.Time) error {+ cfg, err := loadConfig(now())+ if err != nil {+ return err+ }+ shutdown, err := telemetry.Init(ctx, telemetry.Config{+ Endpoint: envDefault("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel.cua.ai"),+ Protocol: envDefault("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"),+ ServiceName: envDefault("OTEL_SERVICE_NAME", "cyclops-usage-meter"),+ ServiceNamespace: envDefault("OTEL_SERVICE_NAMESPACE", "cyclops-cs"),+ Environment: envDefault("OTEL_ENVIRONMENT", "production"),+ ResourceAttrs: os.Getenv("OTEL_RESOURCE_ATTRIBUTES"),+ })+ if err != nil {+ return fmt.Errorf("initialize telemetry: %w", err)+ }+ defer func() {+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ if err := shutdown(shutdownCtx); err != nil {+ slog.Warn("telemetry shutdown failed")+ }+ }()++ ctx, span := otel.Tracer("cyclops-cs-backend/metering").Start(ctx, "usage_meter.collect_hour")+ defer span.End()+ span.SetAttributes(+ attribute.String("meter.cluster_id", cfg.clusterID),+ attribute.String("meter.hour_start", cfg.hourStart.Format(time.RFC3339)),+ attribute.Int64("meter.minimum_coverage_seconds", int64(cfg.minimumCoverage.Seconds())),+ )++ prometheus, err := metering.NewPrometheusClient(cfg.prometheusURL, nil)+ if err != nil {+ span.SetStatus(codes.Error, "invalid Prometheus configuration")+ return err+ }+ store, err := metering.NewPostgresStore(ctx, cfg.databaseURL)+ if err != nil {+ span.SetStatus(codes.Error, "database initialization failed")+ return err+ }+ defer store.Close()++ started := time.Now()+ result, err := (metering.Collector{+ Prometheus: prometheus,+ Tenants: store,+ Writer: store,+ ClusterID: cfg.clusterID,+ Step: cfg.step,+ MaxSampleValidity: cfg.maxSampleValidity,+ MinimumCoverage: cfg.minimumCoverage,+ }).CollectHour(ctx, cfg.hourStart)+ meter := otel.Meter("cyclops-cs-backend/metering")+ duration, _ := meter.Float64Histogram("cyclops.usage_meter.collect.duration", meteringMetricOptions("Hourly reservation collection duration.")...)+ duration.Record(ctx, time.Since(started).Seconds())+ if err != nil {+ failures, _ := meter.Int64Counter("cyclops.usage_meter.collect.failures")+ failures.Add(ctx, 1)+ span.SetStatus(codes.Error, "hourly reservation collection failed")+ return err+ }+ facts, _ := meter.Int64Counter("cyclops.usage_meter.facts")+ facts.Add(ctx, int64(result.Inserted), attributeMetric("outcome", "inserted"))+ facts.Add(ctx, int64(result.Unchanged), attributeMetric("outcome", "unchanged"))+ coverage, _ := meter.Float64Histogram("cyclops.usage_meter.coverage", meteringMetricOptions("KSM source coverage for a materialized hour.")...)+ coverage.Record(ctx, result.Coverage.Seconds())+ span.SetAttributes(+ attribute.Int("meter.sandboxes_discovered", result.Discovered),+ attribute.Int("meter.facts_inserted", result.Inserted),+ attribute.Int("meter.facts_unchanged", result.Unchanged),+ attribute.Int64("meter.coverage_seconds", int64(result.Coverage.Seconds())),+ )+ slog.Info("hourly reservation metering complete",+ "hour_start", cfg.hourStart.Format(time.RFC3339),+ "discovered", result.Discovered,+ "inserted", result.Inserted,+ "unchanged", result.Unchanged,+ "coverage_seconds", int64(result.Coverage.Seconds()))+ return nil+}++type config struct {+ databaseURL string+ prometheusURL string+ clusterID string+ hourStart time.Time+ step time.Duration+ maxSampleValidity time.Duration+ minimumCoverage time.Duration+}++func loadConfig(now time.Time) (config, error) {+ cfg := config{+ databaseURL: os.Getenv("METER_DATABASE_URL"),+ prometheusURL: envDefault("PROMETHEUS_URL", "http://prometheus.cyclops-cs.svc.cluster.local"),+ clusterID: envDefault("METER_CLUSTER_ID", "kopf-k3s"),+ step: 15 * time.Second,+ maxSampleValidity: time.Minute,+ minimumCoverage: 3570 * time.Second,+ }+ if cfg.databaseURL == "" {+ return config{}, fmt.Errorf("METER_DATABASE_URL is required")+ }+ var err error+ if cfg.step, err = durationEnv("METER_SCRAPE_INTERVAL", cfg.step); err != nil {+ return config{}, err+ }+ if cfg.maxSampleValidity, err = durationEnv("METER_MAX_SAMPLE_VALIDITY", cfg.maxSampleValidity); err != nil {+ return config{}, err+ }+ if cfg.minimumCoverage, err = durationEnv("METER_MINIMUM_COVERAGE", cfg.minimumCoverage); err != nil {+ return config{}, err+ }+ cfg.hourStart = now.UTC().Truncate(time.Hour).Add(-time.Hour)+ if rawHour := os.Getenv("METER_HOUR_START"); rawHour != "" {+ cfg.hourStart, err = time.Parse(time.RFC3339, rawHour)+ if err != nil || !cfg.hourStart.Equal(cfg.hourStart.UTC().Truncate(time.Hour)) {+ return config{}, fmt.Errorf("METER_HOUR_START must be an exact UTC RFC3339 hour")+ }+ }+ return cfg, nil+}++func durationEnv(name string, fallback time.Duration) (time.Duration, error) {+ raw := os.Getenv(name)+ if raw == "" {+ return fallback, nil+ }+ value, err := time.ParseDuration(raw)+ if err != nil || value <= 0 {+ return 0, fmt.Errorf("%s must be a positive duration", name)+ }+ return value, nil+}++func envDefault(name, fallback string) string {+ if value := os.Getenv(name); value != "" {+ return value+ }+ return fallback+}++func meteringMetricOptions(description string) []metric.Float64HistogramOption {+ return []metric.Float64HistogramOption{metric.WithDescription(description), metric.WithUnit("s")}+}++func attributeMetric(key, value string) metric.AddOption {+ return metric.WithAttributes(attribute.String(key, value))+}diff --git a/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql b/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sqlnew file mode 100644index 0000000000..eaf452032f--- /dev/null+++ b/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql@@ -0,0 +1,269 @@+CREATE ROLE billing_meter_owner NOLOGIN INHERIT NOCREATEROLE NOSUPERUSER NOCREATEDB NOREPLICATION NOBYPASSRLS;+CREATE ROLE cyclops_meter_writer LOGIN NOINHERIT NOCREATEROLE NOSUPERUSER NOCREATEDB NOREPLICATION NOBYPASSRLS;+GRANT billing_meter_owner TO CURRENT_USER WITH INHERIT FALSE, SET TRUE;++CREATE SCHEMA billing_meter AUTHORIZATION billing_meter_owner;+REVOKE CREATE ON SCHEMA billing_meter FROM PUBLIC;++SET LOCAL ROLE billing_meter_owner;+CREATE TABLE billing_meter.reservation_hour_collection (+ collection_run_id uuid PRIMARY KEY,+ logical_key text NOT NULL,+ revision integer NOT NULL CHECK (revision > 0),+ cluster_id text NOT NULL,+ hour_start timestamptz NOT NULL,+ hour_end timestamptz NOT NULL,+ covered_seconds numeric(12, 6) NOT NULL CHECK (covered_seconds >= 0 AND covered_seconds <= 3600),+ discovered_sandboxes integer NOT NULL CHECK (discovered_sandboxes >= 0),+ inserted_facts integer NOT NULL CHECK (inserted_facts >= 0),+ unchanged_facts integer NOT NULL CHECK (unchanged_facts >= 0),+ source_sha256 text NOT NULL CHECK (source_sha256 ~ '^[0-9a-f]{64}$'),+ supersedes_collection_run_id uuid REFERENCES billing_meter.reservation_hour_collection(collection_run_id),+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),+ UNIQUE (logical_key, revision),+ CHECK (logical_key <> '' AND length(logical_key) <= 1024),+ CHECK (cluster_id <> ''),+ CHECK (date_trunc('hour', hour_start) = hour_start),+ CHECK (hour_end = hour_start + interval '1 hour'),+ CHECK ((revision = 1 AND supersedes_collection_run_id IS NULL) OR (revision > 1 AND supersedes_collection_run_id IS NOT NULL))+);++CREATE TABLE billing_meter.reservation_hour_fact (+ fact_id uuid PRIMARY KEY,+ logical_key text NOT NULL,+ revision integer NOT NULL CHECK (revision > 0),+ cluster_id text NOT NULL,+ capsule_tenant text NOT NULL,+ namespace text NOT NULL,+ sandbox_uid text NOT NULL,+ sandbox_name text NOT NULL,+ pool_name text NOT NULL,+ runtime text NOT NULL,+ hour_start timestamptz NOT NULL,+ hour_end timestamptz NOT NULL,+ virtual_cpu_core_seconds numeric(20, 6) NOT NULL CHECK (virtual_cpu_core_seconds >= 0),+ virtual_memory_byte_seconds numeric(30, 6) NOT NULL CHECK (virtual_memory_byte_seconds >= 0),+ ready_seconds numeric(12, 6) NOT NULL CHECK (ready_seconds >= 0 AND ready_seconds <= 3600),+ covered_seconds numeric(12, 6) NOT NULL CHECK (covered_seconds >= 0 AND covered_seconds <= 3600),+ scrape_interval_seconds integer NOT NULL CHECK (scrape_interval_seconds > 0 AND scrape_interval_seconds <= 300),+ source_sha256 text NOT NULL CHECK (source_sha256 ~ '^[0-9a-f]{64}$'),+ collection_run_id uuid NOT NULL,+ supersedes_fact_id uuid REFERENCES billing_meter.reservation_hour_fact(fact_id),+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),+ UNIQUE (logical_key, revision),+ CHECK (logical_key <> '' AND length(logical_key) <= 1024),+ CHECK (cluster_id <> '' AND capsule_tenant <> '' AND namespace <> ''),+ CHECK (sandbox_uid <> '' AND sandbox_name <> '' AND pool_name <> '' AND runtime <> ''),+ CHECK (date_trunc('hour', hour_start) = hour_start),+ CHECK (hour_end = hour_start + interval '1 hour'),+ CHECK ((revision = 1 AND supersedes_fact_id IS NULL) OR (revision > 1 AND supersedes_fact_id IS NOT NULL))+);++CREATE INDEX reservation_hour_fact_tenant_hour_idx+ ON billing_meter.reservation_hour_fact (capsule_tenant, hour_start, pool_name);+CREATE INDEX reservation_hour_fact_sandbox_hour_idx+ ON billing_meter.reservation_hour_fact (cluster_id, sandbox_uid, hour_start, revision DESC);++CREATE VIEW billing_meter.reservation_hour_collection_current AS+SELECT DISTINCT ON (logical_key)+ collection_run_id,+ logical_key,+ revision,+ cluster_id,+ hour_start,+ hour_end,+ covered_seconds,+ discovered_sandboxes,+ inserted_facts,+ unchanged_facts,+ source_sha256,+ supersedes_collection_run_id,+ created_at+FROM billing_meter.reservation_hour_collection+ORDER BY logical_key, revision DESC;++CREATE VIEW billing_meter.reservation_hour_current AS+SELECT DISTINCT ON (logical_key)+ fact_id,+ logical_key,+ revision,+ cluster_id,+ capsule_tenant,+ namespace,+ sandbox_uid,+ sandbox_name,+ pool_name,+ runtime,+ hour_start,+ hour_end,+ virtual_cpu_core_seconds,+ virtual_memory_byte_seconds,+ ready_seconds,+ covered_seconds,+ scrape_interval_seconds,+ source_sha256,+ collection_run_id,+ supersedes_fact_id,+ created_at+FROM billing_meter.reservation_hour_fact+ORDER BY logical_key, revision DESC;++CREATE FUNCTION billing_meter.reject_reservation_hour_mutation()+RETURNS trigger+LANGUAGE plpgsql+AS $$+BEGIN+ RAISE EXCEPTION 'billing_meter reservation hours are append-only';+END+$$;++CREATE TRIGGER reservation_hour_collection_immutable+BEFORE UPDATE OR DELETE OR TRUNCATE ON billing_meter.reservation_hour_collection+FOR EACH STATEMENT EXECUTE FUNCTION billing_meter.reject_reservation_hour_mutation();+CREATE TRIGGER reservation_hour_fact_immutable+BEFORE UPDATE OR DELETE OR TRUNCATE ON billing_meter.reservation_hour_fact+FOR EACH STATEMENT EXECUTE FUNCTION billing_meter.reject_reservation_hour_mutation();++REVOKE ALL ON ALL TABLES IN SCHEMA billing_meter FROM PUBLIC;+REVOKE ALL ON ALL FUNCTIONS IN SCHEMA billing_meter FROM PUBLIC;+GRANT USAGE ON SCHEMA billing_meter TO cyclops_meter_writer, k8s_reporting_owner;+GRANT SELECT, INSERT ON TABLE billing_meter.reservation_hour_collection, billing_meter.reservation_hour_fact TO cyclops_meter_writer;+GRANT SELECT ON TABLE billing_meter.reservation_hour_collection_current, billing_meter.reservation_hour_current TO cyclops_meter_writer, k8s_reporting_owner;+RESET ROLE;++SET LOCAL ROLE k8s_state_owner;+CREATE FUNCTION k8s_api.sandbox_meter_tenant(+ p_cluster_id text,+ p_namespace text,+ p_sandbox_uid text+)+RETURNS text+LANGUAGE plpgsql+STABLE+SECURITY DEFINER+SET search_path = k8s_state, pg_catalog+AS $$+DECLARE+ resolved_tenant text;+BEGIN+ IF p_cluster_id IS NULL OR p_cluster_id = '' OR p_namespace IS NULL OR p_namespace = '' OR p_sandbox_uid IS NULL OR p_sandbox_uid = '' THEN+ RAISE EXCEPTION 'cluster, namespace, and sandbox UID are required';+ END IF;++ SELECT event.capsule_tenant+ INTO resolved_tenant+ FROM k8s_state.resource_event_outbox AS event+ WHERE event.cluster_id = p_cluster_id+ AND event.api_group = 'osgym.cua.ai'+ AND event.resource = 'osgymsandboxes'+ AND event.namespace = p_namespace+ AND event.uid = p_sandbox_uid+ AND event.capsule_tenant IS NOT NULL+ AND event.capsule_tenant <> ''+ ORDER BY event.observed_at DESC, event.observed_sequence DESC, event.event_id DESC+ LIMIT 1;++ IF resolved_tenant IS NULL THEN+ RAISE EXCEPTION 'sandbox tenant identity is unavailable';+ END IF;+ RETURN resolved_tenant;+END+$$;+REVOKE ALL ON FUNCTION k8s_api.sandbox_meter_tenant(text, text, text) FROM PUBLIC;+GRANT USAGE ON SCHEMA k8s_api TO cyclops_meter_writer;+GRANT EXECUTE ON FUNCTION k8s_api.sandbox_meter_tenant(text, text, text) TO cyclops_meter_writer;+RESET ROLE;++SET LOCAL ROLE k8s_reporting_owner;+CREATE FUNCTION k8s_reporting.reservation_hour_facts(+ p_capsule_tenant text,+ p_start timestamptz,+ p_end timestamptz+)+RETURNS TABLE (+ namespace text,+ sandbox_uid text,+ sandbox_name text,+ pool_name text,+ runtime text,+ hour_start timestamptz,+ hour_end timestamptz,+ virtual_cpu_core_seconds numeric,+ virtual_memory_byte_seconds numeric,+ ready_seconds numeric,+ covered_seconds numeric+)+LANGUAGE plpgsql+STABLE+SECURITY DEFINER+SET search_path = billing_meter, pg_catalog+AS $$+BEGIN+ IF p_capsule_tenant IS NULL OR p_capsule_tenant = '' THEN+ RAISE EXCEPTION 'capsule tenant is required';+ END IF;+ IF p_start IS NULL OR p_end IS NULL OR p_end <= p_start OR p_end - p_start > interval '31 days' THEN+ RAISE EXCEPTION 'reservation usage window must be positive and at most 31 days';+ END IF;++ RETURN QUERY+ SELECT+ fact.namespace,+ fact.sandbox_uid,+ fact.sandbox_name,+ fact.pool_name,+ fact.runtime,+ fact.hour_start,+ fact.hour_end,+ fact.virtual_cpu_core_seconds,+ fact.virtual_memory_byte_seconds,+ fact.ready_seconds,+ fact.covered_seconds+ FROM billing_meter.reservation_hour_current AS fact+ JOIN billing_meter.reservation_hour_collection_current AS collection+ ON collection.cluster_id = fact.cluster_id+ AND collection.hour_start = fact.hour_start+ WHERE fact.capsule_tenant = p_capsule_tenant+ AND fact.hour_start < p_end+ AND fact.hour_end > p_start+ ORDER BY fact.hour_start, fact.namespace, fact.pool_name, fact.sandbox_uid;+END+$$;+REVOKE ALL ON FUNCTION k8s_reporting.reservation_hour_facts(text, timestamptz, timestamptz) FROM PUBLIC;+GRANT EXECUTE ON FUNCTION k8s_reporting.reservation_hour_facts(text, timestamptz, timestamptz) TO cyclops_usage_reader;++CREATE FUNCTION k8s_reporting.reservation_meter_status(+ p_cluster_id text,+ p_start timestamptz,+ p_end timestamptz+)+RETURNS TABLE (+ data_as_of timestamptz,+ complete boolean+)+LANGUAGE plpgsql+STABLE+SECURITY DEFINER+SET search_path = billing_meter, pg_catalog+AS $$+BEGIN+ IF p_cluster_id IS NULL OR p_cluster_id = '' THEN+ RAISE EXCEPTION 'cluster is required';+ END IF;+ IF p_start IS NULL OR p_end IS NULL OR p_end <= p_start OR p_end - p_start > interval '31 days' OR date_trunc('hour', p_start) <> p_start OR date_trunc('hour', p_end) <> p_end THEN+ RAISE EXCEPTION 'reservation meter window must contain exact UTC hours and be at most 31 days';+ END IF;++ RETURN QUERY+ SELECT+ coalesce(max(collection.hour_end), p_start),+ count(*) = extract(epoch FROM p_end - p_start)::bigint / 3600+ FROM billing_meter.reservation_hour_collection_current AS collection+ WHERE collection.cluster_id = p_cluster_id+ AND collection.hour_start >= p_start+ AND collection.hour_end <= p_end;+END+$$;+REVOKE ALL ON FUNCTION k8s_reporting.reservation_meter_status(text, timestamptz, timestamptz) FROM PUBLIC;+GRANT EXECUTE ON FUNCTION k8s_reporting.reservation_meter_status(text, timestamptz, timestamptz) TO cyclops_usage_reader;+RESET ROLE;diff --git a/libs/fleet/backend/database/migrator.go b/libs/fleet/backend/database/migrator.goindex f1177a53d4..2dc369264d 100644--- a/libs/fleet/backend/database/migrator.go+++ b/libs/fleet/backend/database/migrator.go@@ -54,6 +54,7 @@ type CredentialURLs struct {RoleAdmin stringMetabase stringUsage string+ Meter string}type migrationFile struct {@@ -161,6 +162,7 @@ var expectedCredentialRoles = map[string]string{"role-admin": "k8s_role_admin","metabase": "k8s_metabase","usage": "cyclops_usage_reader",+ "meter": "cyclops_meter_writer",}const selectAppliedMigrationsStatement = `select version, filename, sha256 from cyclops_migrations.applied_migrations order by application_order`@@ -477,6 +479,7 @@ func parseCredentialURLs(urls CredentialURLs) ([]credential, error) {{Name: "role-admin", URL: urls.RoleAdmin},{Name: "metabase", URL: urls.Metabase},{Name: "usage", URL: urls.Usage},+ {Name: "meter", URL: urls.Meter},}credentials := make([]credential, 0, len(inputs))@@ -508,8 +511,10 @@ func staticRoleContracts() []staticRoleContract {{role: "k8s_query_admin", inherit: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},{role: "k8s_role_admin", login: true, createRole: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},{role: "k8s_reporting_owner", inherit: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},+ {role: "billing_meter_owner", inherit: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},{role: "k8s_metabase", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},{role: "cyclops_usage_reader", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},+ {role: "cyclops_meter_writer", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},}}@@ -532,6 +537,7 @@ func staticMembershipContracts(migrationOwner string) []staticMembershipContractreturn []staticMembershipContract{{role: "k8s_state_owner", member: migrationOwner, admin: false, inherit: false, set: true},{role: "k8s_reporting_owner", member: migrationOwner, admin: false, inherit: false, set: true},+ {role: "billing_meter_owner", member: migrationOwner, admin: false, inherit: false, set: true},{role: "k8s_query_tenant", member: "k8s_role_admin", admin: true, inherit: false, set: false},{role: "k8s_query_admin", member: "k8s_reporting_owner", admin: false, inherit: true, set: false},}@@ -705,8 +711,8 @@ func readReportingACLs(ctx context.Context, transaction pgx.Tx) ([]reportingACL,routine.oid,casewhen namespace.nspname = 'k8s_reporting'- and routine.proname = 'usage_sandbox_events'- and routine.proargtypes = '25 1184 1184'::oidvector then 'usage_sandbox_events'+ and routine.proname in ('usage_sandbox_events', 'reservation_hour_facts', 'reservation_meter_status')+ and routine.proargtypes = '25 1184 1184'::oidvector then routine.pronameelse routine.oid::regprocedure::textend,routine.proowner::regrole::text,@@ -724,7 +730,7 @@ func readReportingACLs(ctx context.Context, transaction pgx.Tx) ([]reportingACL,acl.grantee in ('k8s_reporting_owner'::regrole, 'k8s_metabase'::regrole, 'cyclops_usage_reader'::regrole)or (namespace.nspname = 'k8s_reporting'- and routine.proname = 'usage_sandbox_events'+ and routine.proname in ('usage_sandbox_events', 'reservation_hour_facts', 'reservation_meter_status')and routine.proargtypes = '25 1184 1184'::oidvector))@@ -775,10 +781,15 @@ func expectedReportingACLs() []reportingACL {{object: reportingObject{kind: reportingObjectSchema, schema: "k8s_state"}, owner: "k8s_state_owner", privilege: reportingPrivilegeUsage, grantee: "k8s_reporting_owner", grantor: "k8s_state_owner"},{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_state", name: "resource_state"}, owner: "k8s_state_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_reporting_owner", grantor: "k8s_state_owner"},{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_state", name: "resource_event_outbox"}, owner: "k8s_state_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_reporting_owner", grantor: "k8s_state_owner"},+ {object: reportingObject{kind: reportingObjectSchema, schema: "billing_meter"}, owner: "billing_meter_owner", privilege: reportingPrivilegeUsage, grantee: "k8s_reporting_owner", grantor: "billing_meter_owner"},+ {object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_current"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_reporting_owner", grantor: "billing_meter_owner"},+ {object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_collection_current"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_reporting_owner", grantor: "billing_meter_owner"},{object: reportingObject{kind: reportingObjectSchema, schema: "k8s_reporting"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeUsage, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_reporting", name: "current_resources"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},{object: reportingObject{kind: reportingObjectSchema, schema: "k8s_reporting"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeUsage, grantee: "cyclops_usage_reader", grantor: "k8s_reporting_owner"},{object: reportingObject{kind: reportingObjectRoutine}, routineIdentity: "usage_sandbox_events", owner: "k8s_reporting_owner", privilege: reportingPrivilegeExecute, grantee: "cyclops_usage_reader", grantor: "k8s_reporting_owner"},+ {object: reportingObject{kind: reportingObjectRoutine}, routineIdentity: "reservation_hour_facts", owner: "k8s_reporting_owner", privilege: reportingPrivilegeExecute, grantee: "cyclops_usage_reader", grantor: "k8s_reporting_owner"},+ {object: reportingObject{kind: reportingObjectRoutine}, routineIdentity: "reservation_meter_status", owner: "k8s_reporting_owner", privilege: reportingPrivilegeExecute, grantee: "cyclops_usage_reader", grantor: "k8s_reporting_owner"},}}@@ -800,26 +811,28 @@ func containsReportingACL(acls []reportingACL, want reportingACL) bool {}func reconcileUsageRoutineExecute(ctx context.Context, transaction pgx.Tx) error {- var routineOID uint32- if err := transaction.QueryRow(ctx, `- select routine.oid- from pg_proc as routine- join pg_namespace as namespace on namespace.oid = routine.pronamespace- where namespace.nspname = 'k8s_reporting'- and routine.proname = 'usage_sandbox_events'- and routine.proargtypes = '25 1184 1184'::oidvector`).Scan(&routineOID); err != nil {- return fmt.Errorf("resolve usage reporting routine: %w", err)- }- acl := reportingACL{- object: reportingObject{kind: reportingObjectRoutine, routineOID: routineOID},- routineIdentity: "usage_sandbox_events",- owner: "k8s_reporting_owner",- privilege: reportingPrivilegeExecute,- grantee: "cyclops_usage_reader",- grantor: "k8s_reporting_owner",- }- if err := newRuntimeDDL(transaction).grantReportingACL(ctx, acl); err != nil {- return fmt.Errorf("reconcile usage routine execute grant: %w", err)+ for _, routineIdentity := range []string{"usage_sandbox_events", "reservation_hour_facts", "reservation_meter_status"} {+ var routineOID uint32+ if err := transaction.QueryRow(ctx, `+ select routine.oid+ from pg_proc as routine+ join pg_namespace as namespace on namespace.oid = routine.pronamespace+ where namespace.nspname = 'k8s_reporting'+ and routine.proname = $1+ and routine.proargtypes = '25 1184 1184'::oidvector`, routineIdentity).Scan(&routineOID); err != nil {+ return fmt.Errorf("resolve usage reporting routine %s: %w", routineIdentity, err)+ }+ acl := reportingACL{+ object: reportingObject{kind: reportingObjectRoutine, routineOID: routineOID},+ routineIdentity: routineIdentity,+ owner: "k8s_reporting_owner",+ privilege: reportingPrivilegeExecute,+ grantee: "cyclops_usage_reader",+ grantor: "k8s_reporting_owner",+ }+ if err := newRuntimeDDL(transaction).grantReportingACL(ctx, acl); err != nil {+ return fmt.Errorf("reconcile usage routine %s execute grant: %w", routineIdentity, err)+ }}return nil}diff --git a/libs/fleet/backend/database/migrator_pg_test.go b/libs/fleet/backend/database/migrator_pg_test.goindex 3f18a9d148..a43fd59ccf 100644--- a/libs/fleet/backend/database/migrator_pg_test.go+++ b/libs/fleet/backend/database/migrator_pg_test.go@@ -24,6 +24,7 @@ const tenantCredentialFingerprintLookup = `select credential_fingerprint from k8var staticMigrationRoles = []string{"cyclops_app","cyclops_usage_reader",+ "cyclops_meter_writer","k8s_state_owner","k8s_state_writer","k8s_state_exporter",@@ -31,6 +32,7 @@ var staticMigrationRoles = []string{"k8s_query_admin","k8s_role_admin","k8s_reporting_owner",+ "billing_meter_owner","k8s_metabase",}@@ -1501,6 +1503,7 @@ func testCredentialURLs(t *testing.T, adminURL string) CredentialURLs {RoleAdmin: withRole("k8s_role_admin"),Metabase: withRole("k8s_metabase"),Usage: withRole("cyclops_usage_reader"),+ Meter: withRole("cyclops_meter_writer"),}}@@ -1678,8 +1681,10 @@ func assertRoleContract(t *testing.T, ctx context.Context, adminURL string) {"k8s_query_admin": {false, true, false},"k8s_role_admin": {true, false, true},"k8s_reporting_owner": {false, true, false},+ "billing_meter_owner": {false, true, false},"k8s_metabase": {true, false, false},"cyclops_usage_reader": {true, false, false},+ "cyclops_meter_writer": {true, false, false},}for role, want := range expected {var login, inherit, createRole, super, createDB, replication, bypassRLS bool@@ -2310,10 +2315,15 @@ func assertExactReportingACLContract(t *testing.T, ctx context.Context, connecti('schema:k8s_state', 'USAGE', 'k8s_reporting_owner', 'k8s_state_owner', false),('relation:k8s_state.resource_state', 'SELECT', 'k8s_reporting_owner', 'k8s_state_owner', false),('relation:k8s_state.resource_event_outbox', 'SELECT', 'k8s_reporting_owner', 'k8s_state_owner', false),+ ('schema:billing_meter', 'USAGE', 'k8s_reporting_owner', 'billing_meter_owner', false),+ ('relation:billing_meter.reservation_hour_current', 'SELECT', 'k8s_reporting_owner', 'billing_meter_owner', false),+ ('relation:billing_meter.reservation_hour_collection_current', 'SELECT', 'k8s_reporting_owner', 'billing_meter_owner', false),('schema:k8s_reporting', 'USAGE', 'k8s_metabase', 'k8s_reporting_owner', false),('relation:k8s_reporting.current_resources', 'SELECT', 'k8s_metabase', 'k8s_reporting_owner', false),('schema:k8s_reporting', 'USAGE', 'cyclops_usage_reader', 'k8s_reporting_owner', false),- ('routine:k8s_reporting.usage_sandbox_events(text,timestamp with time zone,timestamp with time zone)', 'EXECUTE', 'cyclops_usage_reader', 'k8s_reporting_owner', false)+ ('routine:k8s_reporting.usage_sandbox_events(text,timestamp with time zone,timestamp with time zone)', 'EXECUTE', 'cyclops_usage_reader', 'k8s_reporting_owner', false),+ ('routine:k8s_reporting.reservation_hour_facts(text,timestamp with time zone,timestamp with time zone)', 'EXECUTE', 'cyclops_usage_reader', 'k8s_reporting_owner', false),+ ('routine:k8s_reporting.reservation_meter_status(text,timestamp with time zone,timestamp with time zone)', 'EXECUTE', 'cyclops_usage_reader', 'k8s_reporting_owner', false))select count(*) from ((select * from actual except select * from expected)@@ -2349,7 +2359,7 @@ func assertExactReportingACLContract(t *testing.T, ctx context.Context, connectijoin pg_namespace as namespace on namespace.oid = routine.pronamespacejoin lateral aclexplode(routine.proacl) as acl on truewhere namespace.nspname = 'k8s_reporting'- and routine.proname = 'usage_sandbox_events'+ and routine.proname in ('usage_sandbox_events', 'reservation_hour_facts', 'reservation_meter_status')and routine.proargtypes = '25 1184 1184'::oidvectorand acl.grantee <> routine.proownerand (acl.grantee <> 'cyclops_usage_reader'::regrole or acl.privilege_type <> 'EXECUTE' or acl.is_grantable)diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.goindex ff3973dd0e..fac1cf362d 100644--- a/libs/fleet/backend/database/migrator_test.go+++ b/libs/fleet/backend/database/migrator_test.go@@ -520,8 +520,8 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {if err != nil {t.Fatal(err)}- if len(files) != 4 {- t.Fatalf("expected exactly four migrations, got %d", len(files))+ if len(files) != 5 {+ t.Fatalf("expected exactly five migrations, got %d", len(files))}initial := files[0]if initial.Version != 1 || initial.Name != "000001_initial_schema.sql" {@@ -609,6 +609,26 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {t.Errorf("usage filter migration is missing contract %q", expected)}}++ meter := files[4]+ if meter.Version != 5 || meter.Name != "000005_hourly_reservation_meter.sql" {+ t.Fatalf("expected version 5 reservation meter migration, got version=%d name=%q", meter.Version, meter.Name)+ }+ for _, expected := range []string{+ "CREATE ROLE billing_meter_owner NOLOGIN",+ "CREATE ROLE cyclops_meter_writer LOGIN",+ "CREATE TABLE billing_meter.reservation_hour_fact",+ "CREATE VIEW billing_meter.reservation_hour_current",+ "BEFORE UPDATE OR DELETE OR TRUNCATE ON billing_meter.reservation_hour_fact",+ "CREATE FUNCTION k8s_api.sandbox_meter_tenant",+ "CREATE FUNCTION k8s_reporting.reservation_hour_facts",+ "GRANT SELECT, INSERT ON TABLE billing_meter.reservation_hour_collection, billing_meter.reservation_hour_fact TO cyclops_meter_writer",+ "GRANT EXECUTE ON FUNCTION k8s_reporting.reservation_hour_facts(text, timestamptz, timestamptz) TO cyclops_usage_reader",+ } {+ if !strings.Contains(meter.SQL, expected) {+ t.Errorf("reservation meter migration is missing contract %q", expected)+ }+ }}func TestAppliedMigrationLedgerUsesApplicationOrder(t *testing.T) {@@ -726,12 +746,13 @@ func TestCredentialURLsExcludeDynamicTenantRoles(t *testing.T) {RoleAdmin: "postgres://k8s_role_admin:pw@db/cyclops",Metabase: "postgres://k8s_metabase:pw@db/cyclops",Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",+ Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",})if err != nil {t.Fatalf("parse fixed runtime credentials: %v", err)}- if len(credentials) != 6 {- t.Fatalf("credential count = %d, want 6", len(credentials))+ if len(credentials) != 7 {+ t.Fatalf("credential count = %d, want 7", len(credentials))}for _, credential := range credentials {if credential.Role == "k8s_query_broker" || strings.HasPrefix(credential.Role, "k8s_tenant_") {@@ -755,8 +776,10 @@ func TestStaticRoleContractsAreAllowlistedAndFixed(t *testing.T) {"k8s_query_admin": {false, true, false, false, -1, staticRoleValidUntilInfinity},"k8s_role_admin": {true, false, true, false, -1, staticRoleValidUntilInfinity},"k8s_reporting_owner": {false, true, false, false, -1, staticRoleValidUntilInfinity},+ "billing_meter_owner": {false, true, false, false, -1, staticRoleValidUntilInfinity},"k8s_metabase": {true, false, false, false, -1, staticRoleValidUntilInfinity},"cyclops_usage_reader": {true, false, false, false, -1, staticRoleValidUntilInfinity},+ "cyclops_meter_writer": {true, false, false, false, -1, staticRoleValidUntilInfinity},}if len(contracts) != len(want) {t.Fatalf("static role contract count = %d, want %d", len(contracts), len(want))@@ -781,6 +804,7 @@ func TestStaticMembershipContractsUsePG16Options(t *testing.T) {want := []staticMembershipContract{{role: "k8s_state_owner", member: "migration_owner", admin: false, inherit: false, set: true},{role: "k8s_reporting_owner", member: "migration_owner", admin: false, inherit: false, set: true},+ {role: "billing_meter_owner", member: "migration_owner", admin: false, inherit: false, set: true},{role: "k8s_query_tenant", member: "k8s_role_admin", admin: true, inherit: false, set: false},{role: "k8s_query_admin", member: "k8s_reporting_owner", admin: false, inherit: true, set: false},}@@ -805,6 +829,7 @@ func TestCredentialURLsRequireExpectedRoleNames(t *testing.T) {RoleAdmin: "postgres://k8s_role_admin:pw@db/cyclops",Metabase: "postgres://k8s_metabase:pw@db/cyclops",Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",+ Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",})if err == nil {t.Fatal("expected application role-name validation")@@ -822,6 +847,7 @@ func TestCredentialURLsRejectEmptyPassword(t *testing.T) {RoleAdmin: "postgres://k8s_role_admin:pw@db/cyclops",Metabase: "postgres://k8s_metabase:pw@db/cyclops",Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",+ Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",})if err == nil || err.Error() != "application credential database URL must include a password" {t.Fatalf("expected empty password rejection, got %v", err)@@ -836,6 +862,7 @@ func TestCredentialURLParseErrorsPreserveCauseForBoundaryClassification(t *testiRoleAdmin: "postgres://k8s_role_admin:pw@db/cyclops",Metabase: "postgres://k8s_metabase:pw@db/cyclops",Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",+ Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",})if !errors.Is(err, ErrInvalidConfiguration) {t.Fatalf("error = %v, want ErrInvalidConfiguration", err)diff --git a/libs/fleet/backend/database/test-setup.sh b/libs/fleet/backend/database/test-setup.shindex 7390be1555..10886bd802 100755--- a/libs/fleet/backend/database/test-setup.sh+++ b/libs/fleet/backend/database/test-setup.sh@@ -22,8 +22,10 @@ static_roles=(k8s_query_admink8s_role_admink8s_reporting_owner+ billing_meter_ownerk8s_metabasecyclops_usage_reader+ cyclops_meter_writer)base_url="${admin_url%%\?*}"@@ -84,6 +86,7 @@ exporter_url="$(url_for k8s_state_exporter state-exporter-password)"role_admin_url="$(url_for k8s_role_admin role-admin-password)"metabase_url="$(url_for k8s_metabase metabase-password)"usage_url="$(url_for cyclops_usage_reader usage-reader-password)"+meter_url="$(url_for cyclops_meter_writer meter-writer-password)"env_file="${CYCLOPS_TEST_DATABASE_ENV_FILE:-${GITHUB_ENV:-}}"if [[ -z "$env_file" ]]; then@@ -153,6 +156,7 @@ STATE_EXPORTER_DATABASE_URL="$exporter_url" \STATE_ROLE_ADMIN_DATABASE_URL="$role_admin_url" \METABASE_DATABASE_URL="$metabase_url" \USAGE_DATABASE_URL="$usage_url" \+METER_DATABASE_URL="$meter_url" \go run ./cmd/db-migratecat >> "$env_file" <<EOF_ENVdiff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.goindex 6240db0409..99b827ce5a 100644--- a/libs/fleet/backend/main.go+++ b/libs/fleet/backend/main.go@@ -614,5 +614,5 @@ func initializeUsageProvider(ctx context.Context, cfg config.UsageConfiguration)events.Close()return nil, noop, fmt.Errorf("create DataFusion allocation client: %w", err)}- return usage.NewProvider(events, allocations, nil), events.Close, nil+ return usage.NewProviderWithReservations(events, allocations, events, cfg.QueryCluster, nil), events.Close, nil}diff --git a/libs/fleet/backend/metering/collector.go b/libs/fleet/backend/metering/collector.gonew file mode 100644index 0000000000..de09398974--- /dev/null+++ b/libs/fleet/backend/metering/collector.go@@ -0,0 +1,334 @@+package metering++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "fmt"+ "math"+ "sort"+ "strconv"+ "strings"+ "time"++ "github.com/google/uuid"+)++const (+ CPUQuery = "cua_osgym_sandbox_virtual_cpu_cores"+ MemoryQuery = "cua_osgym_sandbox_virtual_memory_bytes"+ ReadyQuery = "cua_osgym_sandbox_ready"+ SourceQuery = `up{job="kube-state-metrics-required"}`+)++type RangeQuerier interface {+ QueryRange(context.Context, string, time.Time, time.Time, time.Duration) ([]Series, error)+}++type TenantResolver interface {+ ResolveTenant(context.Context, string, string, string) (string, error)+}++type FactWriter interface {+ AppendFact(context.Context, HourFact) (bool, error)+ CompleteHour(context.Context, HourCompletion) (bool, error)+}++type HourFact struct {+ FactID uuid.UUID+ LogicalKey string+ ClusterID string+ CapsuleTenant string+ Namespace string+ SandboxUID string+ SandboxName string+ PoolName string+ Runtime string+ HourStart time.Time+ HourEnd time.Time+ VirtualCPUCoreSeconds float64+ VirtualMemoryByteSeconds float64+ ReadySeconds float64+ CoveredSeconds float64+ ScrapeIntervalSeconds int+ SourceSHA256 string+ CollectionRunID uuid.UUID+}++type HourCompletion struct {+ CollectionRunID uuid.UUID+ LogicalKey string+ ClusterID string+ HourStart time.Time+ HourEnd time.Time+ CoveredSeconds float64+ DiscoveredSandboxes int+ InsertedFacts int+ UnchangedFacts int+ SourceSHA256 string+}++type Collector struct {+ Prometheus RangeQuerier+ Tenants TenantResolver+ Writer FactWriter+ ClusterID string+ Step time.Duration+ MaxSampleValidity time.Duration+ MinimumCoverage time.Duration+}++type CollectResult struct {+ Discovered int+ Inserted int+ Unchanged int+ Coverage time.Duration+}++type sandboxIdentity struct {+ Namespace string+ SandboxUID string+ SandboxName string+ PoolName string+ Runtime string+}++type sandboxSamples struct {+ identity sandboxIdentity+ cpu []Sample+ memory []Sample+ ready []Sample+}++func (c Collector) CollectHour(ctx context.Context, hourStart time.Time) (CollectResult, error) {+ if c.Prometheus == nil || c.Tenants == nil || c.Writer == nil || strings.TrimSpace(c.ClusterID) == "" {+ return CollectResult{}, fmt.Errorf("meter collector is not configured")+ }+ if c.Step <= 0 || c.MaxSampleValidity <= 0 || c.MinimumCoverage <= 0 {+ return CollectResult{}, fmt.Errorf("meter collector timing is invalid")+ }+ hourStart = hourStart.UTC()+ if !hourStart.Equal(hourStart.Truncate(time.Hour)) {+ return CollectResult{}, fmt.Errorf("meter hour must start on a UTC hour")+ }+ hourEnd := hourStart.Add(time.Hour)+ queryStart := hourStart.Add(-c.MaxSampleValidity)++ sourceSeries, err := c.Prometheus.QueryRange(ctx, SourceQuery, queryStart, hourEnd, c.Step)+ if err != nil {+ return CollectResult{}, fmt.Errorf("query KSM source coverage: %w", err)+ }+ if len(sourceSeries) != 1 {+ return CollectResult{}, fmt.Errorf("KSM source coverage returned %d series, want 1", len(sourceSeries))+ }+ sourceIntegral, err := Integrate(sourceSeries[0].Samples, hourStart, hourEnd, c.MaxSampleValidity)+ if err != nil {+ return CollectResult{}, fmt.Errorf("integrate KSM source coverage: %w", err)+ }+ coverage := time.Duration(sourceIntegral.ValueSeconds * float64(time.Second))+ if coverage < c.MinimumCoverage {+ return CollectResult{}, fmt.Errorf("KSM source coverage %s is below required %s", coverage, c.MinimumCoverage)+ }++ metricSeries := make(map[string][]Series, 3)+ for _, query := range []string{CPUQuery, MemoryQuery, ReadyQuery} {+ series, queryErr := c.Prometheus.QueryRange(ctx, query, queryStart, hourEnd, c.Step)+ if queryErr != nil {+ return CollectResult{}, fmt.Errorf("query %s: %w", query, queryErr)+ }+ metricSeries[query] = series+ }++ sandboxes, err := mergeSandboxSeries(metricSeries)+ if err != nil {+ return CollectResult{}, err+ }+ result := CollectResult{Discovered: len(sandboxes), Coverage: coverage}+ collectionRunID := uuid.New()+ factHashes := make([]string, 0, len(sandboxes))+ keys := make([]string, 0, len(sandboxes))+ for key := range sandboxes {+ keys = append(keys, key)+ }+ sort.Strings(keys)++ for _, key := range keys {+ sandbox := sandboxes[key]+ cpu, err := Integrate(sandbox.cpu, hourStart, hourEnd, c.MaxSampleValidity)+ if err != nil {+ return CollectResult{}, fmt.Errorf("integrate CPU for sandbox %s: %w", sandbox.identity.SandboxUID, err)+ }+ memory, err := Integrate(sandbox.memory, hourStart, hourEnd, c.MaxSampleValidity)+ if err != nil {+ return CollectResult{}, fmt.Errorf("integrate memory for sandbox %s: %w", sandbox.identity.SandboxUID, err)+ }+ if cpu.ValueSeconds < 0 || memory.ValueSeconds < 0 {+ return CollectResult{}, fmt.Errorf("sandbox %s has negative reservation samples", sandbox.identity.SandboxUID)+ }+ if math.Abs(cpu.PresentSeconds-memory.PresentSeconds) > c.Step.Seconds() {+ return CollectResult{}, fmt.Errorf("sandbox %s CPU and memory sample coverage diverged", sandbox.identity.SandboxUID)+ }+ ready, err := Integrate(sandbox.ready, hourStart, hourEnd, c.MaxSampleValidity)+ if err != nil {+ return CollectResult{}, fmt.Errorf("integrate readiness for sandbox %s: %w", sandbox.identity.SandboxUID, err)+ }+ tenant, err := c.Tenants.ResolveTenant(ctx, c.ClusterID, sandbox.identity.Namespace, sandbox.identity.SandboxUID)+ if err != nil {+ return CollectResult{}, fmt.Errorf("resolve tenant for sandbox %s: %w", sandbox.identity.SandboxUID, err)+ }+ fact := HourFact{+ FactID: uuid.New(),+ LogicalKey: c.ClusterID + "/" + sandbox.identity.SandboxUID + "/" + hourStart.Format(time.RFC3339),+ ClusterID: c.ClusterID,+ CapsuleTenant: tenant,+ Namespace: sandbox.identity.Namespace,+ SandboxUID: sandbox.identity.SandboxUID,+ SandboxName: sandbox.identity.SandboxName,+ PoolName: sandbox.identity.PoolName,+ Runtime: sandbox.identity.Runtime,+ HourStart: hourStart,+ HourEnd: hourEnd,+ VirtualCPUCoreSeconds: cpu.ValueSeconds,+ VirtualMemoryByteSeconds: memory.ValueSeconds,+ ReadySeconds: ready.ValueSeconds,+ CoveredSeconds: sourceIntegral.ValueSeconds,+ ScrapeIntervalSeconds: int(c.Step.Seconds()),+ CollectionRunID: collectionRunID,+ }+ fact.SourceSHA256 = hashFactSource(fact, sandbox, sourceSeries[0].Samples)+ factHashes = append(factHashes, fact.SourceSHA256)+ inserted, err := c.Writer.AppendFact(ctx, fact)+ if err != nil {+ return CollectResult{}, fmt.Errorf("append sandbox %s reservation fact: %w", sandbox.identity.SandboxUID, err)+ }+ if inserted {+ result.Inserted+++ } else {+ result.Unchanged+++ }+ }+ sort.Strings(factHashes)+ completion := HourCompletion{+ CollectionRunID: collectionRunID,+ LogicalKey: c.ClusterID + "/" + hourStart.Format(time.RFC3339),+ ClusterID: c.ClusterID,+ HourStart: hourStart,+ HourEnd: hourEnd,+ CoveredSeconds: sourceIntegral.ValueSeconds,+ DiscoveredSandboxes: result.Discovered,+ InsertedFacts: result.Inserted,+ UnchangedFacts: result.Unchanged,+ SourceSHA256: hashCollectionSource(c.ClusterID, hourStart, sourceSeries[0].Samples, factHashes),+ }+ if _, err := c.Writer.CompleteHour(ctx, completion); err != nil {+ return CollectResult{}, fmt.Errorf("complete reservation hour: %w", err)+ }+ return result, nil+}++func mergeSandboxSeries(metrics map[string][]Series) (map[string]*sandboxSamples, error) {+ out := make(map[string]*sandboxSamples)+ for _, metric := range []string{CPUQuery, MemoryQuery, ReadyQuery} {+ for _, series := range metrics[metric] {+ identity, err := identityFromLabels(series.Labels)+ if err != nil {+ return nil, fmt.Errorf("invalid %s labels: %w", metric, err)+ }+ key := identity.Namespace + "/" + identity.SandboxUID+ sandbox := out[key]+ if sandbox == nil {+ sandbox = &sandboxSamples{identity: identity}+ out[key] = sandbox+ } else if sandbox.identity != identity {+ return nil, fmt.Errorf("sandbox %s metric labels disagree", identity.SandboxUID)+ }+ switch metric {+ case CPUQuery:+ if sandbox.cpu != nil {+ return nil, fmt.Errorf("sandbox %s has duplicate CPU series", identity.SandboxUID)+ }+ sandbox.cpu = series.Samples+ case MemoryQuery:+ if sandbox.memory != nil {+ return nil, fmt.Errorf("sandbox %s has duplicate memory series", identity.SandboxUID)+ }+ sandbox.memory = series.Samples+ case ReadyQuery:+ if sandbox.ready != nil {+ return nil, fmt.Errorf("sandbox %s has duplicate ready series", identity.SandboxUID)+ }+ sandbox.ready = series.Samples+ }+ }+ }+ for _, sandbox := range out {+ if sandbox.cpu == nil || sandbox.memory == nil {+ return nil, fmt.Errorf("sandbox %s is missing CPU or memory series", sandbox.identity.SandboxUID)+ }+ }+ return out, nil+}++func identityFromLabels(labels map[string]string) (sandboxIdentity, error) {+ poolName := strings.TrimSpace(labels["pool"])+ if poolName == "" {+ poolName = strings.TrimSpace(labels["warmpool"])+ }+ identity := sandboxIdentity{+ Namespace: strings.TrimSpace(labels["namespace"]),+ SandboxUID: strings.TrimSpace(labels["sandbox_uid"]),+ SandboxName: strings.TrimSpace(labels["sandbox"]),+ PoolName: poolName,+ Runtime: strings.TrimSpace(labels["runtime"]),+ }+ if identity.Namespace == "" || identity.SandboxUID == "" || identity.SandboxName == "" || identity.PoolName == "" || identity.Runtime == "" {+ return sandboxIdentity{}, fmt.Errorf("required identity label is empty")+ }+ return identity, nil+}++func hashFactSource(fact HourFact, sandbox *sandboxSamples, source []Sample) string {+ hash := sha256.New()+ for _, value := range []string{+ fact.LogicalKey,+ fact.Namespace,+ fact.SandboxName,+ fact.PoolName,+ fact.Runtime,+ strconv.Itoa(fact.ScrapeIntervalSeconds),+ } {+ _, _ = hash.Write([]byte(value))+ _, _ = hash.Write([]byte{0})+ }+ for _, samples := range [][]Sample{sandbox.cpu, sandbox.memory, sandbox.ready, source} {+ ordered := append([]Sample(nil), samples...)+ sort.Slice(ordered, func(i, j int) bool { return ordered[i].Timestamp.Before(ordered[j].Timestamp) })+ for _, sample := range ordered {+ _, _ = hash.Write([]byte(strconv.FormatInt(sample.Timestamp.UnixNano(), 10)))+ _, _ = hash.Write([]byte{'='})+ _, _ = hash.Write([]byte(strconv.FormatFloat(sample.Value, 'g', -1, 64)))+ _, _ = hash.Write([]byte{0})+ }+ _, _ = hash.Write([]byte{0xff})+ }+ return hex.EncodeToString(hash.Sum(nil))+}++func hashCollectionSource(clusterID string, hourStart time.Time, source []Sample, factHashes []string) string {+ hash := sha256.New()+ _, _ = hash.Write([]byte(clusterID))+ _, _ = hash.Write([]byte{0})+ _, _ = hash.Write([]byte(hourStart.Format(time.RFC3339)))+ _, _ = hash.Write([]byte{0})+ for _, sample := range source {+ _, _ = hash.Write([]byte(strconv.FormatInt(sample.Timestamp.UnixNano(), 10)))+ _, _ = hash.Write([]byte{'='})+ _, _ = hash.Write([]byte(strconv.FormatFloat(sample.Value, 'g', -1, 64)))+ _, _ = hash.Write([]byte{0})+ }+ for _, factHash := range factHashes {+ _, _ = hash.Write([]byte(factHash))+ _, _ = hash.Write([]byte{0})+ }+ return hex.EncodeToString(hash.Sum(nil))+}diff --git a/libs/fleet/backend/metering/collector_test.go b/libs/fleet/backend/metering/collector_test.gonew file mode 100644index 0000000000..2c835a7100--- /dev/null+++ b/libs/fleet/backend/metering/collector_test.go@@ -0,0 +1,113 @@+package metering++import (+ "context"+ "strings"+ "testing"+ "time"+)++type fakeQuerier map[string][]Series++func (f fakeQuerier) QueryRange(_ context.Context, query string, _, _ time.Time, _ time.Duration) ([]Series, error) {+ return f[query], nil+}++type fakeTenantResolver struct{ tenant string }++func (f fakeTenantResolver) ResolveTenant(context.Context, string, string, string) (string, error) {+ return f.tenant, nil+}++type fakeFactWriter struct{ facts []HourFact }++func (f *fakeFactWriter) AppendFact(_ context.Context, fact HourFact) (bool, error) {+ f.facts = append(f.facts, fact)+ return true, nil+}++func (f *fakeFactWriter) CompleteHour(context.Context, HourCompletion) (bool, error) {+ return true, nil+}++func TestCollectorMaterializesReservationHour(t *testing.T) {+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)+ labels := map[string]string{+ "namespace": "tenant-a",+ "sandbox": "sandbox-a",+ "sandbox_uid": "uid-a",+ "pool": "pool-a",+ "runtime": "kubevirt",+ }+ writer := &fakeFactWriter{}+ collector := Collector{+ Prometheus: fakeQuerier{+ SourceQuery: {{Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, 1)}},+ CPUQuery: {{Labels: labels, Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, 4)}},+ MemoryQuery: {{Labels: labels, Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, float64(8<<30))}},+ ReadyQuery: {{Labels: labels, Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, 1)}},+ },+ Tenants: fakeTenantResolver{tenant: "personal:user-a"},+ Writer: writer,+ ClusterID: "kopf-k3s",+ Step: 15 * time.Second,+ MaxSampleValidity: time.Minute,+ MinimumCoverage: 3570 * time.Second,+ }++ result, err := collector.CollectHour(context.Background(), start)+ if err != nil {+ t.Fatal(err)+ }+ if result.Discovered != 1 || result.Inserted != 1 || len(writer.facts) != 1 {+ t.Fatalf("result = %+v, facts = %d", result, len(writer.facts))+ }+ fact := writer.facts[0]+ if fact.VirtualCPUCoreSeconds != 4*3600 || fact.VirtualMemoryByteSeconds != float64(8<<30)*3600 || fact.ReadySeconds != 3600 || fact.CoveredSeconds != 3600 {+ t.Fatalf("fact quantities = %+v", fact)+ }+ if fact.CapsuleTenant != "personal:user-a" || fact.SourceSHA256 == "" || !strings.Contains(fact.LogicalKey, "uid-a") {+ t.Fatalf("fact identity = %+v", fact)+ }+}++func TestCollectorFailsClosedOnSourceCoverage(t *testing.T) {+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)+ collector := Collector{+ Prometheus: fakeQuerier{SourceQuery: {{Samples: []Sample{{Timestamp: start, Value: 1}}}}},+ Tenants: fakeTenantResolver{tenant: "personal:user-a"},+ Writer: &fakeFactWriter{},+ ClusterID: "kopf-k3s",+ Step: 15 * time.Second,+ MaxSampleValidity: time.Minute,+ MinimumCoverage: 3570 * time.Second,+ }+ _, err := collector.CollectHour(context.Background(), start)+ if err == nil || !strings.Contains(err.Error(), "below required") {+ t.Fatalf("error = %v, want coverage rejection", err)+ }+}++func TestIdentityFromLabelsFallsBackToWarmPoolLabel(t *testing.T) {+ identity, err := identityFromLabels(map[string]string{+ "namespace": "tenant-a",+ "sandbox": "sandbox-a",+ "sandbox_uid": "uid-a",+ "warmpool": "pool-a",+ "runtime": "macos",+ })+ if err != nil {+ t.Fatal(err)+ }+ if identity.PoolName != "pool-a" {+ t.Fatalf("pool = %q, want pool-a", identity.PoolName)+ }+}++func constantSamples(start, end time.Time, step time.Duration, value float64) []Sample {+ var samples []Sample+ for timestamp := start; !timestamp.After(end); timestamp = timestamp.Add(step) {+ samples = append(samples, Sample{Timestamp: timestamp, Value: value})+ }+ return samples+}diff --git a/libs/fleet/backend/metering/integrate.go b/libs/fleet/backend/metering/integrate.gonew file mode 100644index 0000000000..3e0695f3e2--- /dev/null+++ b/libs/fleet/backend/metering/integrate.go@@ -0,0 +1,57 @@+package metering++import (+ "fmt"+ "math"+ "sort"+ "time"+)++type Sample struct {+ Timestamp time.Time+ Value float64+}++type Integral struct {+ ValueSeconds float64+ PresentSeconds float64+}++func Integrate(samples []Sample, start, end time.Time, maxValidity time.Duration) (Integral, error) {+ if start.IsZero() || end.IsZero() || !end.After(start) || maxValidity <= 0 {+ return Integral{}, fmt.Errorf("invalid integration window")+ }++ ordered := append([]Sample(nil), samples...)+ sort.Slice(ordered, func(i, j int) bool { return ordered[i].Timestamp.Before(ordered[j].Timestamp) })++ var result Integral+ for index, sample := range ordered {+ if sample.Timestamp.IsZero() || math.IsNaN(sample.Value) || math.IsInf(sample.Value, 0) {+ return Integral{}, fmt.Errorf("invalid metric sample")+ }+ if index > 0 && !sample.Timestamp.After(ordered[index-1].Timestamp) {+ return Integral{}, fmt.Errorf("metric samples must have unique timestamps")+ }++ intervalStart := sample.Timestamp+ if intervalStart.Before(start) {+ intervalStart = start+ }+ intervalEnd := sample.Timestamp.Add(maxValidity)+ if index+1 < len(ordered) && ordered[index+1].Timestamp.Before(intervalEnd) {+ intervalEnd = ordered[index+1].Timestamp+ }+ if intervalEnd.After(end) {+ intervalEnd = end+ }+ if !intervalEnd.After(intervalStart) || !intervalStart.Before(end) {+ continue+ }++ seconds := intervalEnd.Sub(intervalStart).Seconds()+ result.ValueSeconds += sample.Value * seconds+ result.PresentSeconds += seconds+ }+ return result, nil+}diff --git a/libs/fleet/backend/metering/integrate_test.go b/libs/fleet/backend/metering/integrate_test.gonew file mode 100644index 0000000000..201ef5164d--- /dev/null+++ b/libs/fleet/backend/metering/integrate_test.go@@ -0,0 +1,40 @@+package metering++import (+ "math"+ "testing"+ "time"+)++func TestIntegrateUsesPreviousSampleAtWindowBoundary(t *testing.T) {+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)+ result, err := Integrate([]Sample{+ {Timestamp: start.Add(-15 * time.Second), Value: 4},+ {Timestamp: start.Add(15 * time.Second), Value: 4},+ }, start, start.Add(30*time.Second), time.Minute)+ if err != nil {+ t.Fatal(err)+ }+ if result.ValueSeconds != 120 || result.PresentSeconds != 30 {+ t.Fatalf("integral = %+v, want 120 value-seconds over 30 seconds", result)+ }+}++func TestIntegrateCapsStaleSamples(t *testing.T) {+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)+ result, err := Integrate([]Sample{{Timestamp: start, Value: 8}}, start, start.Add(time.Hour), time.Minute)+ if err != nil {+ t.Fatal(err)+ }+ if result.ValueSeconds != 480 || result.PresentSeconds != 60 {+ t.Fatalf("integral = %+v, want one minute of validity", result)+ }+}++func TestIntegrateRejectsNonFiniteSamples(t *testing.T) {+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)+ _, err := Integrate([]Sample{{Timestamp: start, Value: math.NaN()}}, start, start.Add(time.Hour), time.Minute)+ if err == nil {+ t.Fatal("expected non-finite sample rejection")+ }+}diff --git a/libs/fleet/backend/metering/postgres.go b/libs/fleet/backend/metering/postgres.gonew file mode 100644index 0000000000..47c31a29ab--- /dev/null+++ b/libs/fleet/backend/metering/postgres.go@@ -0,0 +1,189 @@+package metering++import (+ "context"+ "errors"+ "fmt"+ "strings"+ "time"++ "github.com/google/uuid"+ "github.com/jackc/pgx/v5"+ "github.com/jackc/pgx/v5/pgxpool"+)++type PostgresStore struct {+ pool *pgxpool.Pool+}++func NewPostgresStore(ctx context.Context, databaseURL string) (*PostgresStore, error) {+ config, err := pgxpool.ParseConfig(databaseURL)+ if err != nil {+ return nil, fmt.Errorf("parse meter database URL: %w", err)+ }+ if config.ConnConfig.User != "cyclops_meter_writer" {+ return nil, fmt.Errorf("meter database URL must use cyclops_meter_writer")+ }+ config.MaxConns = 2+ config.MinConns = 0+ config.MaxConnIdleTime = time.Minute+ config.MaxConnLifetime = 15 * time.Minute+ pool, err := pgxpool.NewWithConfig(ctx, config)+ if err != nil {+ return nil, fmt.Errorf("open meter database pool: %w", err)+ }+ return &PostgresStore{pool: pool}, nil+}++func (s *PostgresStore) Close() {+ if s != nil && s.pool != nil {+ s.pool.Close()+ }+}++func (s *PostgresStore) ResolveTenant(ctx context.Context, clusterID, namespace, sandboxUID string) (string, error) {+ var tenant string+ if err := s.pool.QueryRow(ctx, `select k8s_api.sandbox_meter_tenant($1, $2, $3)`, clusterID, namespace, sandboxUID).Scan(&tenant); err != nil {+ return "", fmt.Errorf("query sandbox tenant: %w", err)+ }+ if strings.TrimSpace(tenant) == "" {+ return "", fmt.Errorf("sandbox tenant is empty")+ }+ return tenant, nil+}++func (s *PostgresStore) AppendFact(ctx context.Context, fact HourFact) (bool, error) {+ if err := validateHourFact(fact); err != nil {+ return false, err+ }+ transaction, err := s.pool.BeginTx(ctx, pgx.TxOptions{})+ if err != nil {+ return false, fmt.Errorf("begin fact transaction: %w", err)+ }+ defer func() { _ = transaction.Rollback(ctx) }()+ if _, err := transaction.Exec(ctx, `select pg_advisory_xact_lock(hashtextextended($1, 0))`, fact.LogicalKey); err != nil {+ return false, fmt.Errorf("lock reservation fact: %w", err)+ }++ var previousID uuid.UUID+ var previousRevision int+ var previousSHA string+ err = transaction.QueryRow(ctx, `select fact_id, revision, source_sha256+ from billing_meter.reservation_hour_current where logical_key = $1`, fact.LogicalKey).+ Scan(&previousID, &previousRevision, &previousSHA)+ if err == nil && previousSHA == fact.SourceSHA256 {+ if err := transaction.Commit(ctx); err != nil {+ return false, fmt.Errorf("commit unchanged reservation fact: %w", err)+ }+ return false, nil+ }+ if err != nil && !errors.Is(err, pgx.ErrNoRows) {+ return false, fmt.Errorf("read current reservation fact: %w", err)+ }++ revision := 1+ var supersedes any+ if err == nil {+ revision = previousRevision + 1+ supersedes = previousID+ }+ _, err = transaction.Exec(ctx, `insert into billing_meter.reservation_hour_fact (+ fact_id, logical_key, revision, cluster_id, capsule_tenant, namespace,+ sandbox_uid, sandbox_name, pool_name, runtime, hour_start, hour_end,+ virtual_cpu_core_seconds, virtual_memory_byte_seconds, ready_seconds,+ covered_seconds, scrape_interval_seconds, source_sha256, collection_run_id,+ supersedes_fact_id+ ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)`,+ fact.FactID, fact.LogicalKey, revision, fact.ClusterID, fact.CapsuleTenant, fact.Namespace,+ fact.SandboxUID, fact.SandboxName, fact.PoolName, fact.Runtime, fact.HourStart, fact.HourEnd,+ fact.VirtualCPUCoreSeconds, fact.VirtualMemoryByteSeconds, fact.ReadySeconds,+ fact.CoveredSeconds, fact.ScrapeIntervalSeconds, fact.SourceSHA256, fact.CollectionRunID,+ supersedes)+ if err != nil {+ return false, fmt.Errorf("insert reservation fact: %w", err)+ }+ if err := transaction.Commit(ctx); err != nil {+ return false, fmt.Errorf("commit reservation fact: %w", err)+ }+ return true, nil+}++func (s *PostgresStore) CompleteHour(ctx context.Context, completion HourCompletion) (bool, error) {+ if err := validateHourCompletion(completion); err != nil {+ return false, err+ }+ transaction, err := s.pool.BeginTx(ctx, pgx.TxOptions{})+ if err != nil {+ return false, fmt.Errorf("begin hour completion transaction: %w", err)+ }+ defer func() { _ = transaction.Rollback(ctx) }()+ if _, err := transaction.Exec(ctx, `select pg_advisory_xact_lock(hashtextextended(, 0))`, completion.LogicalKey); err != nil {+ return false, fmt.Errorf("lock reservation hour completion: %w", err)+ }+ var previousID uuid.UUID+ var previousRevision int+ var previousSHA string+ err = transaction.QueryRow(ctx, `select collection_run_id, revision, source_sha256+ from billing_meter.reservation_hour_collection_current where logical_key = `, completion.LogicalKey).+ Scan(&previousID, &previousRevision, &previousSHA)+ if err == nil && previousSHA == completion.SourceSHA256 {+ if err := transaction.Commit(ctx); err != nil {+ return false, fmt.Errorf("commit unchanged hour completion: %w", err)+ }+ return false, nil+ }+ if err != nil && !errors.Is(err, pgx.ErrNoRows) {+ return false, fmt.Errorf("read current hour completion: %w", err)+ }+ revision := 1+ var supersedes any+ if err == nil {+ revision = previousRevision + 1+ supersedes = previousID+ }+ _, err = transaction.Exec(ctx, `insert into billing_meter.reservation_hour_collection (+ collection_run_id, logical_key, revision, cluster_id, hour_start, hour_end,+ covered_seconds, discovered_sandboxes, inserted_facts, unchanged_facts,+ source_sha256, supersedes_collection_run_id+ ) values (,,,,,,,,,,,)`,+ completion.CollectionRunID, completion.LogicalKey, revision, completion.ClusterID,+ completion.HourStart, completion.HourEnd, completion.CoveredSeconds,+ completion.DiscoveredSandboxes, completion.InsertedFacts, completion.UnchangedFacts,+ completion.SourceSHA256, supersedes)+ if err != nil {+ return false, fmt.Errorf("insert hour completion: %w", err)+ }+ if err := transaction.Commit(ctx); err != nil {+ return false, fmt.Errorf("commit hour completion: %w", err)+ }+ return true, nil+}++func validateHourCompletion(completion HourCompletion) error {+ if completion.CollectionRunID == uuid.Nil || completion.LogicalKey == "" || completion.ClusterID == "" || len(completion.SourceSHA256) != 64 {+ return fmt.Errorf("hour completion has empty required fields")+ }+ if !completion.HourEnd.Equal(completion.HourStart.Add(time.Hour)) || !completion.HourStart.Equal(completion.HourStart.UTC().Truncate(time.Hour)) {+ return fmt.Errorf("hour completion has invalid hour")+ }+ if completion.CoveredSeconds < 0 || completion.CoveredSeconds > 3600 || completion.DiscoveredSandboxes < 0 || completion.InsertedFacts < 0 || completion.UnchangedFacts < 0 {+ return fmt.Errorf("hour completion has invalid quantities")+ }+ return nil+}++func validateHourFact(fact HourFact) error {+ if fact.FactID == uuid.Nil || fact.CollectionRunID == uuid.Nil || fact.LogicalKey == "" || fact.ClusterID == "" || fact.CapsuleTenant == "" || fact.Namespace == "" || fact.SandboxUID == "" || fact.SandboxName == "" || fact.PoolName == "" || fact.Runtime == "" {+ return fmt.Errorf("reservation fact has empty required fields")+ }+ if !fact.HourEnd.Equal(fact.HourStart.Add(time.Hour)) || !fact.HourStart.Equal(fact.HourStart.UTC().Truncate(time.Hour)) {+ return fmt.Errorf("reservation fact has invalid hour")+ }+ if len(fact.SourceSHA256) != 64 || fact.ScrapeIntervalSeconds <= 0 || fact.VirtualCPUCoreSeconds < 0 || fact.VirtualMemoryByteSeconds < 0 || fact.ReadySeconds < 0 || fact.CoveredSeconds < 0 || fact.CoveredSeconds > 3600 {+ return fmt.Errorf("reservation fact has invalid quantities")+ }+ return nil+}++var _ TenantResolver = (*PostgresStore)(nil)+var _ FactWriter = (*PostgresStore)(nil)diff --git a/libs/fleet/backend/metering/prometheus.go b/libs/fleet/backend/metering/prometheus.gonew file mode 100644index 0000000000..7dcdea5327--- /dev/null+++ b/libs/fleet/backend/metering/prometheus.go@@ -0,0 +1,126 @@+package metering++import (+ "context"+ "encoding/json"+ "fmt"+ "io"+ "net/http"+ "net/url"+ "strconv"+ "strings"+ "time"+)++const maxPrometheusResponseBytes = 32 << 20++type Series struct {+ Labels map[string]string+ Samples []Sample+}++type PrometheusClient struct {+ baseURL *url.URL+ client *http.Client+}++func NewPrometheusClient(rawURL string, client *http.Client) (*PrometheusClient, error) {+ baseURL, err := url.Parse(rawURL)+ if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" {+ return nil, fmt.Errorf("invalid Prometheus URL")+ }+ if client == nil {+ client = &http.Client{Timeout: 30 * time.Second}+ }+ return &PrometheusClient{baseURL: baseURL, client: client}, nil+}++func (c *PrometheusClient) QueryRange(ctx context.Context, query string, start, end time.Time, step time.Duration) ([]Series, error) {+ if strings.TrimSpace(query) == "" || !end.After(start) || step <= 0 {+ return nil, fmt.Errorf("invalid Prometheus range query")+ }+ endpoint := *c.baseURL+ endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/api/v1/query_range"+ values := endpoint.Query()+ values.Set("query", query)+ values.Set("start", formatPrometheusTime(start))+ values.Set("end", formatPrometheusTime(end))+ values.Set("step", strconv.FormatFloat(step.Seconds(), 'f', -1, 64))+ endpoint.RawQuery = values.Encode()++ request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)+ if err != nil {+ return nil, fmt.Errorf("build Prometheus request: %w", err)+ }+ response, err := c.client.Do(request)+ if err != nil {+ return nil, fmt.Errorf("query Prometheus: %w", err)+ }+ defer response.Body.Close()+ body, err := io.ReadAll(io.LimitReader(response.Body, maxPrometheusResponseBytes+1))+ if err != nil {+ return nil, fmt.Errorf("read Prometheus response: %w", err)+ }+ if len(body) > maxPrometheusResponseBytes {+ return nil, fmt.Errorf("Prometheus response exceeds %d bytes", maxPrometheusResponseBytes)+ }+ if response.StatusCode != http.StatusOK {+ return nil, fmt.Errorf("Prometheus returned HTTP %d", response.StatusCode)+ }++ var payload struct {+ Status string `json:"status"`+ Data struct {+ ResultType string `json:"resultType"`+ Result []struct {+ Metric map[string]string `json:"metric"`+ Values [][]json.RawMessage `json:"values"`+ } `json:"result"`+ } `json:"data"`+ Error string `json:"error"`+ }+ if err := json.Unmarshal(body, &payload); err != nil {+ return nil, fmt.Errorf("decode Prometheus response: %w", err)+ }+ if payload.Status != "success" || payload.Data.ResultType != "matrix" {+ return nil, fmt.Errorf("Prometheus range query failed")+ }++ series := make([]Series, 0, len(payload.Data.Result))+ for _, result := range payload.Data.Result {+ parsed := Series{Labels: result.Metric, Samples: make([]Sample, 0, len(result.Values))}+ for _, pair := range result.Values {+ if len(pair) != 2 {+ return nil, fmt.Errorf("invalid Prometheus sample pair")+ }+ var timestamp float64+ var valueText string+ if err := json.Unmarshal(pair[0], ×tamp); err != nil {+ return nil, fmt.Errorf("decode Prometheus timestamp: %w", err)+ }+ if err := json.Unmarshal(pair[1], &valueText); err != nil {+ return nil, fmt.Errorf("decode Prometheus value: %w", err)+ }+ value, err := strconv.ParseFloat(valueText, 64)+ if err != nil {+ return nil, fmt.Errorf("parse Prometheus value: %w", err)+ }+ seconds, fraction := mathModf(timestamp)+ parsed.Samples = append(parsed.Samples, Sample{+ Timestamp: time.Unix(seconds, int64(fraction*float64(time.Second))).UTC(),+ Value: value,+ })+ }+ series = append(series, parsed)+ }+ return series, nil+}++func formatPrometheusTime(value time.Time) string {+ return strconv.FormatFloat(float64(value.UnixNano())/float64(time.Second), 'f', 3, 64)+}++func mathModf(value float64) (int64, float64) {+ seconds := int64(value)+ return seconds, value - float64(seconds)+}diff --git a/libs/fleet/backend/metering/prometheus_test.go b/libs/fleet/backend/metering/prometheus_test.gonew file mode 100644index 0000000000..9127de9056--- /dev/null+++ b/libs/fleet/backend/metering/prometheus_test.go@@ -0,0 +1,35 @@+package metering++import (+ "context"+ "net/http"+ "net/http/httptest"+ "testing"+ "time"+)++func TestPrometheusClientParsesMatrix(t *testing.T) {+ server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {+ if request.URL.Path != "/api/v1/query_range" || request.URL.Query().Get("query") != "metric_name" {+ t.Fatalf("unexpected request: %s", request.URL.String())+ }+ response.Header().Set("Content-Type", "application/json")+ _, _ = response.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"sandbox_uid":"uid-1"},"values":[[1787382000.5,"4"]]}]}}`))+ }))+ defer server.Close()++ client, err := NewPrometheusClient(server.URL, server.Client())+ if err != nil {+ t.Fatal(err)+ }+ series, err := client.QueryRange(context.Background(), "metric_name", time.Unix(1787382000, 0), time.Unix(1787382060, 0), 15*time.Second)+ if err != nil {+ t.Fatal(err)+ }+ if len(series) != 1 || series[0].Labels["sandbox_uid"] != "uid-1" || len(series[0].Samples) != 1 || series[0].Samples[0].Value != 4 {+ t.Fatalf("series = %#v", series)+ }+ if got := series[0].Samples[0].Timestamp.Nanosecond(); got != 500000000 {+ t.Fatalf("timestamp nanos = %d, want 500000000", got)+ }+}diff --git a/libs/fleet/backend/usage/postgres.go b/libs/fleet/backend/usage/postgres.goindex 77b2e5d882..0b6e389ba8 100644--- a/libs/fleet/backend/usage/postgres.go+++ b/libs/fleet/backend/usage/postgres.go@@ -94,3 +94,38 @@ func validateSandboxEvent(event SandboxEvent) error {}var _ EventStore = (*PostgresEventStore)(nil)++const reservationFactsQuery = `select namespace, sandbox_uid, sandbox_name, pool_name, runtime,+ hour_start, hour_end, virtual_cpu_core_seconds, virtual_memory_byte_seconds,+ ready_seconds, covered_seconds+from k8s_reporting.reservation_hour_facts($1, $2, $3)`++func (s *PostgresEventStore) Reservations(ctx context.Context, tenant, cluster string, start, end time.Time) ([]ReservationFact, time.Time, bool, error) {+ if strings.TrimSpace(tenant) == "" || strings.TrimSpace(cluster) == "" || start.IsZero() || end.IsZero() || !end.After(start) {+ return nil, time.Time{}, false, fmt.Errorf("invalid reservation query")+ }+ var asOf time.Time+ var complete bool+ if err := s.pool.QueryRow(ctx, `select data_as_of, complete from k8s_reporting.reservation_meter_status(, , )`, cluster, start, end).Scan(&asOf, &complete); err != nil {+ return nil, time.Time{}, false, fmt.Errorf("query reservation meter status: %w", err)+ }+ rows, err := s.pool.Query(ctx, reservationFactsQuery, tenant, start, end)+ if err != nil {+ return nil, time.Time{}, false, fmt.Errorf("query reservation facts: %w", err)+ }+ defer rows.Close()+ facts := make([]ReservationFact, 0)+ for rows.Next() {+ var fact ReservationFact+ if err := rows.Scan(&fact.Namespace, &fact.SandboxUID, &fact.SandboxName, &fact.PoolName, &fact.Runtime, &fact.HourStart, &fact.HourEnd, &fact.VirtualCPUCoreSeconds, &fact.VirtualMemoryByteSeconds, &fact.ReadySeconds, &fact.CoveredSeconds); err != nil {+ return nil, time.Time{}, false, fmt.Errorf("scan reservation fact: %w", err)+ }+ facts = append(facts, fact)+ }+ if err := rows.Err(); err != nil {+ return nil, time.Time{}, false, fmt.Errorf("read reservation facts: %w", err)+ }+ return facts, asOf.UTC(), !complete, nil+}++var _ ReservationStore = (*PostgresEventStore)(nil)diff --git a/libs/fleet/backend/usage/provider.go b/libs/fleet/backend/usage/provider.goindex 5a35f25e20..eb3fca84ac 100644--- a/libs/fleet/backend/usage/provider.go+++ b/libs/fleet/backend/usage/provider.go@@ -27,13 +27,15 @@ const ()type provider struct {- events EventStore- allocations AllocationClient- clock func() time.Time- cacheTTL time.Duration- cacheMu sync.Mutex- cache map[usageCacheKey]usageCacheEntry- loads singleflight.Group+ events EventStore+ allocations AllocationClient+ reservations ReservationStore+ reservationCluster string+ clock func() time.Time+ cacheTTL time.Duration+ cacheMu sync.Mutex+ cache map[usageCacheKey]usageCacheEntry+ loads singleflight.Group}type usageCacheKey struct {@@ -58,11 +60,15 @@ type liveSegment struct {}func NewProvider(events EventStore, allocations AllocationClient, clock func() time.Time) *provider {+ return NewProviderWithReservations(events, allocations, nil, "", clock)+}++func NewProviderWithReservations(events EventStore, allocations AllocationClient, reservations ReservationStore, reservationCluster string, clock func() time.Time) *provider {if clock == nil {clock = time.Now}return &provider{- events: events, allocations: allocations, clock: clock,+ events: events, allocations: allocations, reservations: reservations, reservationCluster: reservationCluster, clock: clock,cacheTTL: defaultUsageCacheTTL, cache: make(map[usageCacheKey]usageCacheEntry),}}@@ -269,6 +275,26 @@ func (p *provider) loadUncached(ctx context.Context, query Query, now time.Time)data.partial = true}}+ if p.reservations != nil {+ reservationCtx, reservationSpan := usageTracer().Start(ctx, "usage.reservations.query")+ facts, reservationAsOf, reservationPartial, reservationErr := p.reservations.Reservations(reservationCtx, identity.PersonalGroup(reservationCtx, query.Subject), p.reservationCluster, start, end)+ if reservationErr != nil {+ markUsageSpanError(reservationSpan, "reservation fact query failed")+ reservationSpan.End()+ markUsageSpanError(span, "reservation fact query failed")+ return usageData{}, fmt.Errorf("read reservation facts: %w", reservationErr)+ }+ reservationSpan.SetAttributes(attribute.Int("usage.reservation_fact_count", len(facts)))+ reservationSpan.End()+ if reservationAsOf.Before(data.asOf) {+ data.asOf = maxTime(start, reservationAsOf.UTC())+ }+ data.partial = data.partial || reservationPartial || data.asOf.Before(end)+ if err := applyReservationFacts(facts, start, end, step, poolTotals, data.buckets); err != nil {+ markUsageSpanError(span, "reservation fact attribution failed")+ return usageData{}, err+ }+ }for _, totals := range poolTotals {if !totals.isZero() {data.pools = append(data.pools, totals)@@ -641,3 +667,35 @@ func maxTime(left, right time.Time) time.Time {}var _ Provider = (*provider)(nil)++func applyReservationFacts(facts []ReservationFact, windowStart, windowEnd time.Time, bucketDuration time.Duration, pools map[string]usageTotals, buckets map[bucketKey]usageTotals) error {+ for id, pool := range pools {+ pool.cpuProvisioned = 0+ pool.memoryProvisioned = 0+ pools[id] = pool+ }+ for key, bucket := range buckets {+ bucket.cpuProvisioned = 0+ bucket.memoryProvisioned = 0+ buckets[key] = bucket+ }+ for _, fact := range facts {+ if fact.Namespace == "" || !validPoolName(fact.PoolName) || !fact.HourEnd.Equal(fact.HourStart.Add(time.Hour)) || fact.HourStart.Before(windowStart) || fact.HourEnd.After(windowEnd) || fact.VirtualCPUCoreSeconds < 0 || fact.VirtualMemoryByteSeconds < 0 {+ return fmt.Errorf("invalid reservation fact")+ }+ cpuCoreHours := fact.VirtualCPUCoreSeconds / 3600+ memoryGiBHours := fact.VirtualMemoryByteSeconds / 3600 / gibibyte+ id := fact.Namespace + ":" + fact.PoolName+ pool := pools[id]+ pool.id, pool.name = id, fact.PoolName+ pool.cpuProvisioned += cpuCoreHours+ pool.memoryProvisioned += memoryGiBHours+ pools[id] = pool+ key := bucketKey{namespace: fact.Namespace, pool: fact.PoolName, start: bucketStartFor(fact.HourStart, windowStart, bucketDuration)}+ bucket := buckets[key]+ bucket.cpuProvisioned += cpuCoreHours+ bucket.memoryProvisioned += memoryGiBHours+ buckets[key] = bucket+ }+ return nil+}diff --git a/libs/fleet/backend/usage/provider_test.go b/libs/fleet/backend/usage/provider_test.goindex a886ded87c..a67722f483 100644--- a/libs/fleet/backend/usage/provider_test.go+++ b/libs/fleet/backend/usage/provider_test.go@@ -388,3 +388,49 @@ func TestProviderRestoresAttributionAfterUnambiguousAdded(t *testing.T) {t.Fatalf("overview=%#v err=%v", got, err)}}++type fakeReservationStore struct {+ facts []ReservationFact+ asOf time.Time+ partial bool+ err error+ tenant string+ cluster string+}++func (s *fakeReservationStore) Reservations(_ context.Context, tenant, cluster string, _, _ time.Time) ([]ReservationFact, time.Time, bool, error) {+ s.tenant, s.cluster = tenant, cluster+ return s.facts, s.asOf, s.partial, s.err+}++func TestProviderUsesVirtualReservationsForProvisionedTotals(t *testing.T) {+ t.Parallel()+ cutoff := time.Date(2026, 8, 22, 8, 0, 0, 0, time.UTC)+ events := &fakeEventStore{events: []SandboxEvent{{+ EventID: mustUUID("00000000-0000-0000-0000-000000000090"), Namespace: "ns-a", SandboxName: "sandbox-a", SandboxUID: "uid-a",+ PoolName: "pool-a", Runtime: "kubevirt", VMName: "vm-a", EventType: "Added", ObservedAt: cutoff.Add(-25 * time.Hour),+ }}}+ allocations := &fakeAllocationClient{asOf: cutoff, allocations: []Allocation{{+ Start: cutoff.Add(-time.Hour), End: cutoff, Namespace: "ns-a", Pod: "virt-launcher-vm-a-abc",+ Minutes: 60, CPUUsageAverage: 1, CPURequestAverage: 2,+ RAMUsageAverageBytes: 3 * gibibyte, RAMRequestAverageBytes: 4 * gibibyte, CostUSD: 0.5,+ }}}+ reservations := &fakeReservationStore{asOf: cutoff, facts: []ReservationFact{{+ Namespace: "ns-a", SandboxUID: "uid-a", SandboxName: "sandbox-a", PoolName: "pool-a", Runtime: "kubevirt",+ HourStart: cutoff.Add(-time.Hour), HourEnd: cutoff,+ VirtualCPUCoreSeconds: 4 * 3600, VirtualMemoryByteSeconds: 8 * gibibyte * 3600,+ }}}+ provider := NewProviderWithReservations(events, allocations, reservations, "kopf-k3s", func() time.Time { return cutoff })++ got, err := provider.Overview(context.Background(), Query{Subject: "alice", Timeframe: Timeframe24H})+ if err != nil {+ t.Fatal(err)+ }+ if reservations.tenant != "user-alice" || reservations.cluster != "kopf-k3s" || got.Partial || len(got.Pools) != 1 {+ t.Fatalf("overview=%#v reservations=%+v", got, reservations)+ }+ pool := got.Pools[0]+ if pool.CPU.Consumed != 1 || pool.CPU.Provisioned != 4 || pool.Memory.Consumed != 3 || pool.Memory.Provisioned != 8 || pool.CostUSD != 0.5 {+ t.Fatalf("pool=%#v", pool)+ }+}diff --git a/libs/fleet/backend/usage/types.go b/libs/fleet/backend/usage/types.goindex d8bd0464fe..7fce917ef7 100644--- a/libs/fleet/backend/usage/types.go+++ b/libs/fleet/backend/usage/types.go@@ -112,3 +112,21 @@ type AllocationClient interface {}const gibibyte = float64(1 << 30)++type ReservationFact struct {+ Namespace string+ SandboxUID string+ SandboxName string+ PoolName string+ Runtime string+ HourStart time.Time+ HourEnd time.Time+ VirtualCPUCoreSeconds float64+ VirtualMemoryByteSeconds float64+ ReadySeconds float64+ CoveredSeconds float64+}++type ReservationStore interface {+ Reservations(context.Context, string, string, time.Time, time.Time) ([]ReservationFact, time.Time, bool, error)+}