Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(cyclops): materialize hourly sandbox reservations (#7150)

r33drichards268723a1db4e
diff --git a/libs/fleet/backend/Dockerfile b/libs/fleet/backend/Dockerfile
index 7137b23aa0..28b726bdc9 100644
--- a/libs/fleet/backend/Dockerfile
+++ b/libs/fleet/backend/Dockerfile
@@ -10,7 +10,8 @@ RUN go mod tidy
RUN go install github.com/swaggo/swag/cmd/[email protected] && \
/go/bin/swag init -g main.go -o ./docs --parseDependency --parseInternal
RUN CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-cs-backend . \
- && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-db-migrate ./cmd/db-migrate
+ && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-db-migrate ./cmd/db-migrate \
+ && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-usage-meter ./cmd/usage-meter
FROM --platform=linux/amd64 rust:1.94-bookworm AS rust-builder
WORKDIR /src/k8s-state-query
@@ -23,6 +24,7 @@ FROM gcr.io/distroless/cc-debian12:nonroot AS runtime
WORKDIR /app
COPY --from=go-builder /out/cyclops-cs-backend /app/cyclops-cs-backend
COPY --from=go-builder /out/cyclops-db-migrate /app/cyclops-db-migrate
+COPY --from=go-builder /out/cyclops-usage-meter /app/cyclops-usage-meter
COPY --from=rust-builder /out-k8s-state-projector /app/k8s-state-projector
USER nonroot:nonroot
EXPOSE 8080
diff --git a/libs/fleet/backend/cmd/db-migrate/main.go b/libs/fleet/backend/cmd/db-migrate/main.go
index 8417bc9b48..bb061c3593 100644
--- a/libs/fleet/backend/cmd/db-migrate/main.go
+++ b/libs/fleet/backend/cmd/db-migrate/main.go
@@ -36,6 +36,7 @@ func main() {
RoleAdmin: required("STATE_ROLE_ADMIN_DATABASE_URL"),
Metabase: required("METABASE_DATABASE_URL"),
Usage: required("USAGE_DATABASE_URL"),
+ Meter: required("METER_DATABASE_URL"),
},
}
if err := database.Run(context.Background(), config); err != nil {
diff --git a/libs/fleet/backend/cmd/usage-meter/main.go b/libs/fleet/backend/cmd/usage-meter/main.go
new file mode 100644
index 0000000000..df54aa9b52
--- /dev/null
+++ b/libs/fleet/backend/cmd/usage-meter/main.go
@@ -0,0 +1,176 @@
+package main
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "os"
+ "time"
+
+ "cyclops-cs-backend/metering"
+ "cyclops-cs-backend/telemetry"
+ "go.opentelemetry.io/otel"
+ "go.opentelemetry.io/otel/attribute"
+ "go.opentelemetry.io/otel/codes"
+ "go.opentelemetry.io/otel/metric"
+)
+
+func main() {
+ slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))
+ if err := run(context.Background(), time.Now); err != nil {
+ slog.Error("hourly reservation metering failed", "error", err)
+ os.Exit(1)
+ }
+}
+
+func run(ctx context.Context, now func() time.Time) error {
+ cfg, err := loadConfig(now())
+ if err != nil {
+ return err
+ }
+ shutdown, err := telemetry.Init(ctx, telemetry.Config{
+ Endpoint: envDefault("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel.cua.ai"),
+ Protocol: envDefault("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"),
+ ServiceName: envDefault("OTEL_SERVICE_NAME", "cyclops-usage-meter"),
+ ServiceNamespace: envDefault("OTEL_SERVICE_NAMESPACE", "cyclops-cs"),
+ Environment: envDefault("OTEL_ENVIRONMENT", "production"),
+ ResourceAttrs: os.Getenv("OTEL_RESOURCE_ATTRIBUTES"),
+ })
+ if err != nil {
+ return fmt.Errorf("initialize telemetry: %w", err)
+ }
+ defer func() {
+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+ defer cancel()
+ if err := shutdown(shutdownCtx); err != nil {
+ slog.Warn("telemetry shutdown failed")
+ }
+ }()
+
+ ctx, span := otel.Tracer("cyclops-cs-backend/metering").Start(ctx, "usage_meter.collect_hour")
+ defer span.End()
+ span.SetAttributes(
+ attribute.String("meter.cluster_id", cfg.clusterID),
+ attribute.String("meter.hour_start", cfg.hourStart.Format(time.RFC3339)),
+ attribute.Int64("meter.minimum_coverage_seconds", int64(cfg.minimumCoverage.Seconds())),
+ )
+
+ prometheus, err := metering.NewPrometheusClient(cfg.prometheusURL, nil)
+ if err != nil {
+ span.SetStatus(codes.Error, "invalid Prometheus configuration")
+ return err
+ }
+ store, err := metering.NewPostgresStore(ctx, cfg.databaseURL)
+ if err != nil {
+ span.SetStatus(codes.Error, "database initialization failed")
+ return err
+ }
+ defer store.Close()
+
+ started := time.Now()
+ result, err := (metering.Collector{
+ Prometheus: prometheus,
+ Tenants: store,
+ Writer: store,
+ ClusterID: cfg.clusterID,
+ Step: cfg.step,
+ MaxSampleValidity: cfg.maxSampleValidity,
+ MinimumCoverage: cfg.minimumCoverage,
+ }).CollectHour(ctx, cfg.hourStart)
+ meter := otel.Meter("cyclops-cs-backend/metering")
+ duration, _ := meter.Float64Histogram("cyclops.usage_meter.collect.duration", meteringMetricOptions("Hourly reservation collection duration.")...)
+ duration.Record(ctx, time.Since(started).Seconds())
+ if err != nil {
+ failures, _ := meter.Int64Counter("cyclops.usage_meter.collect.failures")
+ failures.Add(ctx, 1)
+ span.SetStatus(codes.Error, "hourly reservation collection failed")
+ return err
+ }
+ facts, _ := meter.Int64Counter("cyclops.usage_meter.facts")
+ facts.Add(ctx, int64(result.Inserted), attributeMetric("outcome", "inserted"))
+ facts.Add(ctx, int64(result.Unchanged), attributeMetric("outcome", "unchanged"))
+ coverage, _ := meter.Float64Histogram("cyclops.usage_meter.coverage", meteringMetricOptions("KSM source coverage for a materialized hour.")...)
+ coverage.Record(ctx, result.Coverage.Seconds())
+ span.SetAttributes(
+ attribute.Int("meter.sandboxes_discovered", result.Discovered),
+ attribute.Int("meter.facts_inserted", result.Inserted),
+ attribute.Int("meter.facts_unchanged", result.Unchanged),
+ attribute.Int64("meter.coverage_seconds", int64(result.Coverage.Seconds())),
+ )
+ slog.Info("hourly reservation metering complete",
+ "hour_start", cfg.hourStart.Format(time.RFC3339),
+ "discovered", result.Discovered,
+ "inserted", result.Inserted,
+ "unchanged", result.Unchanged,
+ "coverage_seconds", int64(result.Coverage.Seconds()))
+ return nil
+}
+
+type config struct {
+ databaseURL string
+ prometheusURL string
+ clusterID string
+ hourStart time.Time
+ step time.Duration
+ maxSampleValidity time.Duration
+ minimumCoverage time.Duration
+}
+
+func loadConfig(now time.Time) (config, error) {
+ cfg := config{
+ databaseURL: os.Getenv("METER_DATABASE_URL"),
+ prometheusURL: envDefault("PROMETHEUS_URL", "http://prometheus.cyclops-cs.svc.cluster.local"),
+ clusterID: envDefault("METER_CLUSTER_ID", "kopf-k3s"),
+ step: 15 * time.Second,
+ maxSampleValidity: time.Minute,
+ minimumCoverage: 3570 * time.Second,
+ }
+ if cfg.databaseURL == "" {
+ return config{}, fmt.Errorf("METER_DATABASE_URL is required")
+ }
+ var err error
+ if cfg.step, err = durationEnv("METER_SCRAPE_INTERVAL", cfg.step); err != nil {
+ return config{}, err
+ }
+ if cfg.maxSampleValidity, err = durationEnv("METER_MAX_SAMPLE_VALIDITY", cfg.maxSampleValidity); err != nil {
+ return config{}, err
+ }
+ if cfg.minimumCoverage, err = durationEnv("METER_MINIMUM_COVERAGE", cfg.minimumCoverage); err != nil {
+ return config{}, err
+ }
+ cfg.hourStart = now.UTC().Truncate(time.Hour).Add(-time.Hour)
+ if rawHour := os.Getenv("METER_HOUR_START"); rawHour != "" {
+ cfg.hourStart, err = time.Parse(time.RFC3339, rawHour)
+ if err != nil || !cfg.hourStart.Equal(cfg.hourStart.UTC().Truncate(time.Hour)) {
+ return config{}, fmt.Errorf("METER_HOUR_START must be an exact UTC RFC3339 hour")
+ }
+ }
+ return cfg, nil
+}
+
+func durationEnv(name string, fallback time.Duration) (time.Duration, error) {
+ raw := os.Getenv(name)
+ if raw == "" {
+ return fallback, nil
+ }
+ value, err := time.ParseDuration(raw)
+ if err != nil || value <= 0 {
+ return 0, fmt.Errorf("%s must be a positive duration", name)
+ }
+ return value, nil
+}
+
+func envDefault(name, fallback string) string {
+ if value := os.Getenv(name); value != "" {
+ return value
+ }
+ return fallback
+}
+
+func meteringMetricOptions(description string) []metric.Float64HistogramOption {
+ return []metric.Float64HistogramOption{metric.WithDescription(description), metric.WithUnit("s")}
+}
+
+func attributeMetric(key, value string) metric.AddOption {
+ return metric.WithAttributes(attribute.String(key, value))
+}
diff --git a/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql b/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql
new file mode 100644
index 0000000000..eaf452032f
--- /dev/null
+++ b/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql
@@ -0,0 +1,269 @@
+CREATE ROLE billing_meter_owner NOLOGIN INHERIT NOCREATEROLE NOSUPERUSER NOCREATEDB NOREPLICATION NOBYPASSRLS;
+CREATE ROLE cyclops_meter_writer LOGIN NOINHERIT NOCREATEROLE NOSUPERUSER NOCREATEDB NOREPLICATION NOBYPASSRLS;
+GRANT billing_meter_owner TO CURRENT_USER WITH INHERIT FALSE, SET TRUE;
+
+CREATE SCHEMA billing_meter AUTHORIZATION billing_meter_owner;
+REVOKE CREATE ON SCHEMA billing_meter FROM PUBLIC;
+
+SET LOCAL ROLE billing_meter_owner;
+CREATE TABLE billing_meter.reservation_hour_collection (
+ collection_run_id uuid PRIMARY KEY,
+ logical_key text NOT NULL,
+ revision integer NOT NULL CHECK (revision > 0),
+ cluster_id text NOT NULL,
+ hour_start timestamptz NOT NULL,
+ hour_end timestamptz NOT NULL,
+ covered_seconds numeric(12, 6) NOT NULL CHECK (covered_seconds >= 0 AND covered_seconds <= 3600),
+ discovered_sandboxes integer NOT NULL CHECK (discovered_sandboxes >= 0),
+ inserted_facts integer NOT NULL CHECK (inserted_facts >= 0),
+ unchanged_facts integer NOT NULL CHECK (unchanged_facts >= 0),
+ source_sha256 text NOT NULL CHECK (source_sha256 ~ '^[0-9a-f]{64}$'),
+ supersedes_collection_run_id uuid REFERENCES billing_meter.reservation_hour_collection(collection_run_id),
+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+ UNIQUE (logical_key, revision),
+ CHECK (logical_key <> '' AND length(logical_key) <= 1024),
+ CHECK (cluster_id <> ''),
+ CHECK (date_trunc('hour', hour_start) = hour_start),
+ CHECK (hour_end = hour_start + interval '1 hour'),
+ CHECK ((revision = 1 AND supersedes_collection_run_id IS NULL) OR (revision > 1 AND supersedes_collection_run_id IS NOT NULL))
+);
+
+CREATE TABLE billing_meter.reservation_hour_fact (
+ fact_id uuid PRIMARY KEY,
+ logical_key text NOT NULL,
+ revision integer NOT NULL CHECK (revision > 0),
+ cluster_id text NOT NULL,
+ capsule_tenant text NOT NULL,
+ namespace text NOT NULL,
+ sandbox_uid text NOT NULL,
+ sandbox_name text NOT NULL,
+ pool_name text NOT NULL,
+ runtime text NOT NULL,
+ hour_start timestamptz NOT NULL,
+ hour_end timestamptz NOT NULL,
+ virtual_cpu_core_seconds numeric(20, 6) NOT NULL CHECK (virtual_cpu_core_seconds >= 0),
+ virtual_memory_byte_seconds numeric(30, 6) NOT NULL CHECK (virtual_memory_byte_seconds >= 0),
+ ready_seconds numeric(12, 6) NOT NULL CHECK (ready_seconds >= 0 AND ready_seconds <= 3600),
+ covered_seconds numeric(12, 6) NOT NULL CHECK (covered_seconds >= 0 AND covered_seconds <= 3600),
+ scrape_interval_seconds integer NOT NULL CHECK (scrape_interval_seconds > 0 AND scrape_interval_seconds <= 300),
+ source_sha256 text NOT NULL CHECK (source_sha256 ~ '^[0-9a-f]{64}$'),
+ collection_run_id uuid NOT NULL,
+ supersedes_fact_id uuid REFERENCES billing_meter.reservation_hour_fact(fact_id),
+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+ UNIQUE (logical_key, revision),
+ CHECK (logical_key <> '' AND length(logical_key) <= 1024),
+ CHECK (cluster_id <> '' AND capsule_tenant <> '' AND namespace <> ''),
+ CHECK (sandbox_uid <> '' AND sandbox_name <> '' AND pool_name <> '' AND runtime <> ''),
+ CHECK (date_trunc('hour', hour_start) = hour_start),
+ CHECK (hour_end = hour_start + interval '1 hour'),
+ CHECK ((revision = 1 AND supersedes_fact_id IS NULL) OR (revision > 1 AND supersedes_fact_id IS NOT NULL))
+);
+
+CREATE INDEX reservation_hour_fact_tenant_hour_idx
+ ON billing_meter.reservation_hour_fact (capsule_tenant, hour_start, pool_name);
+CREATE INDEX reservation_hour_fact_sandbox_hour_idx
+ ON billing_meter.reservation_hour_fact (cluster_id, sandbox_uid, hour_start, revision DESC);
+
+CREATE VIEW billing_meter.reservation_hour_collection_current AS
+SELECT DISTINCT ON (logical_key)
+ collection_run_id,
+ logical_key,
+ revision,
+ cluster_id,
+ hour_start,
+ hour_end,
+ covered_seconds,
+ discovered_sandboxes,
+ inserted_facts,
+ unchanged_facts,
+ source_sha256,
+ supersedes_collection_run_id,
+ created_at
+FROM billing_meter.reservation_hour_collection
+ORDER BY logical_key, revision DESC;
+
+CREATE VIEW billing_meter.reservation_hour_current AS
+SELECT DISTINCT ON (logical_key)
+ fact_id,
+ logical_key,
+ revision,
+ cluster_id,
+ capsule_tenant,
+ namespace,
+ sandbox_uid,
+ sandbox_name,
+ pool_name,
+ runtime,
+ hour_start,
+ hour_end,
+ virtual_cpu_core_seconds,
+ virtual_memory_byte_seconds,
+ ready_seconds,
+ covered_seconds,
+ scrape_interval_seconds,
+ source_sha256,
+ collection_run_id,
+ supersedes_fact_id,
+ created_at
+FROM billing_meter.reservation_hour_fact
+ORDER BY logical_key, revision DESC;
+
+CREATE FUNCTION billing_meter.reject_reservation_hour_mutation()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+BEGIN
+ RAISE EXCEPTION 'billing_meter reservation hours are append-only';
+END
+$$;
+
+CREATE TRIGGER reservation_hour_collection_immutable
+BEFORE UPDATE OR DELETE OR TRUNCATE ON billing_meter.reservation_hour_collection
+FOR EACH STATEMENT EXECUTE FUNCTION billing_meter.reject_reservation_hour_mutation();
+CREATE TRIGGER reservation_hour_fact_immutable
+BEFORE UPDATE OR DELETE OR TRUNCATE ON billing_meter.reservation_hour_fact
+FOR EACH STATEMENT EXECUTE FUNCTION billing_meter.reject_reservation_hour_mutation();
+
+REVOKE ALL ON ALL TABLES IN SCHEMA billing_meter FROM PUBLIC;
+REVOKE ALL ON ALL FUNCTIONS IN SCHEMA billing_meter FROM PUBLIC;
+GRANT USAGE ON SCHEMA billing_meter TO cyclops_meter_writer, k8s_reporting_owner;
+GRANT SELECT, INSERT ON TABLE billing_meter.reservation_hour_collection, billing_meter.reservation_hour_fact TO cyclops_meter_writer;
+GRANT SELECT ON TABLE billing_meter.reservation_hour_collection_current, billing_meter.reservation_hour_current TO cyclops_meter_writer, k8s_reporting_owner;
+RESET ROLE;
+
+SET LOCAL ROLE k8s_state_owner;
+CREATE FUNCTION k8s_api.sandbox_meter_tenant(
+ p_cluster_id text,
+ p_namespace text,
+ p_sandbox_uid text
+)
+RETURNS text
+LANGUAGE plpgsql
+STABLE
+SECURITY DEFINER
+SET search_path = k8s_state, pg_catalog
+AS $$
+DECLARE
+ resolved_tenant text;
+BEGIN
+ IF p_cluster_id IS NULL OR p_cluster_id = '' OR p_namespace IS NULL OR p_namespace = '' OR p_sandbox_uid IS NULL OR p_sandbox_uid = '' THEN
+ RAISE EXCEPTION 'cluster, namespace, and sandbox UID are required';
+ END IF;
+
+ SELECT event.capsule_tenant
+ INTO resolved_tenant
+ FROM k8s_state.resource_event_outbox AS event
+ WHERE event.cluster_id = p_cluster_id
+ AND event.api_group = 'osgym.cua.ai'
+ AND event.resource = 'osgymsandboxes'
+ AND event.namespace = p_namespace
+ AND event.uid = p_sandbox_uid
+ AND event.capsule_tenant IS NOT NULL
+ AND event.capsule_tenant <> ''
+ ORDER BY event.observed_at DESC, event.observed_sequence DESC, event.event_id DESC
+ LIMIT 1;
+
+ IF resolved_tenant IS NULL THEN
+ RAISE EXCEPTION 'sandbox tenant identity is unavailable';
+ END IF;
+ RETURN resolved_tenant;
+END
+$$;
+REVOKE ALL ON FUNCTION k8s_api.sandbox_meter_tenant(text, text, text) FROM PUBLIC;
+GRANT USAGE ON SCHEMA k8s_api TO cyclops_meter_writer;
+GRANT EXECUTE ON FUNCTION k8s_api.sandbox_meter_tenant(text, text, text) TO cyclops_meter_writer;
+RESET ROLE;
+
+SET LOCAL ROLE k8s_reporting_owner;
+CREATE FUNCTION k8s_reporting.reservation_hour_facts(
+ p_capsule_tenant text,
+ p_start timestamptz,
+ p_end timestamptz
+)
+RETURNS TABLE (
+ namespace text,
+ sandbox_uid text,
+ sandbox_name text,
+ pool_name text,
+ runtime text,
+ hour_start timestamptz,
+ hour_end timestamptz,
+ virtual_cpu_core_seconds numeric,
+ virtual_memory_byte_seconds numeric,
+ ready_seconds numeric,
+ covered_seconds numeric
+)
+LANGUAGE plpgsql
+STABLE
+SECURITY DEFINER
+SET search_path = billing_meter, pg_catalog
+AS $$
+BEGIN
+ IF p_capsule_tenant IS NULL OR p_capsule_tenant = '' THEN
+ RAISE EXCEPTION 'capsule tenant is required';
+ END IF;
+ IF p_start IS NULL OR p_end IS NULL OR p_end <= p_start OR p_end - p_start > interval '31 days' THEN
+ RAISE EXCEPTION 'reservation usage window must be positive and at most 31 days';
+ END IF;
+
+ RETURN QUERY
+ SELECT
+ fact.namespace,
+ fact.sandbox_uid,
+ fact.sandbox_name,
+ fact.pool_name,
+ fact.runtime,
+ fact.hour_start,
+ fact.hour_end,
+ fact.virtual_cpu_core_seconds,
+ fact.virtual_memory_byte_seconds,
+ fact.ready_seconds,
+ fact.covered_seconds
+ FROM billing_meter.reservation_hour_current AS fact
+ JOIN billing_meter.reservation_hour_collection_current AS collection
+ ON collection.cluster_id = fact.cluster_id
+ AND collection.hour_start = fact.hour_start
+ WHERE fact.capsule_tenant = p_capsule_tenant
+ AND fact.hour_start < p_end
+ AND fact.hour_end > p_start
+ ORDER BY fact.hour_start, fact.namespace, fact.pool_name, fact.sandbox_uid;
+END
+$$;
+REVOKE ALL ON FUNCTION k8s_reporting.reservation_hour_facts(text, timestamptz, timestamptz) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION k8s_reporting.reservation_hour_facts(text, timestamptz, timestamptz) TO cyclops_usage_reader;
+
+CREATE FUNCTION k8s_reporting.reservation_meter_status(
+ p_cluster_id text,
+ p_start timestamptz,
+ p_end timestamptz
+)
+RETURNS TABLE (
+ data_as_of timestamptz,
+ complete boolean
+)
+LANGUAGE plpgsql
+STABLE
+SECURITY DEFINER
+SET search_path = billing_meter, pg_catalog
+AS $$
+BEGIN
+ IF p_cluster_id IS NULL OR p_cluster_id = '' THEN
+ RAISE EXCEPTION 'cluster is required';
+ END IF;
+ IF p_start IS NULL OR p_end IS NULL OR p_end <= p_start OR p_end - p_start > interval '31 days' OR date_trunc('hour', p_start) <> p_start OR date_trunc('hour', p_end) <> p_end THEN
+ RAISE EXCEPTION 'reservation meter window must contain exact UTC hours and be at most 31 days';
+ END IF;
+
+ RETURN QUERY
+ SELECT
+ coalesce(max(collection.hour_end), p_start),
+ count(*) = extract(epoch FROM p_end - p_start)::bigint / 3600
+ FROM billing_meter.reservation_hour_collection_current AS collection
+ WHERE collection.cluster_id = p_cluster_id
+ AND collection.hour_start >= p_start
+ AND collection.hour_end <= p_end;
+END
+$$;
+REVOKE ALL ON FUNCTION k8s_reporting.reservation_meter_status(text, timestamptz, timestamptz) FROM PUBLIC;
+GRANT EXECUTE ON FUNCTION k8s_reporting.reservation_meter_status(text, timestamptz, timestamptz) TO cyclops_usage_reader;
+RESET ROLE;
diff --git a/libs/fleet/backend/database/migrator.go b/libs/fleet/backend/database/migrator.go
index f1177a53d4..2dc369264d 100644
--- a/libs/fleet/backend/database/migrator.go
+++ b/libs/fleet/backend/database/migrator.go
@@ -54,6 +54,7 @@ type CredentialURLs struct {
RoleAdmin string
Metabase string
Usage string
+ Meter string
}
type migrationFile struct {
@@ -161,6 +162,7 @@ var expectedCredentialRoles = map[string]string{
"role-admin": "k8s_role_admin",
"metabase": "k8s_metabase",
"usage": "cyclops_usage_reader",
+ "meter": "cyclops_meter_writer",
}
const selectAppliedMigrationsStatement = `select version, filename, sha256 from cyclops_migrations.applied_migrations order by application_order`
@@ -477,6 +479,7 @@ func parseCredentialURLs(urls CredentialURLs) ([]credential, error) {
{Name: "role-admin", URL: urls.RoleAdmin},
{Name: "metabase", URL: urls.Metabase},
{Name: "usage", URL: urls.Usage},
+ {Name: "meter", URL: urls.Meter},
}
credentials := make([]credential, 0, len(inputs))
@@ -508,8 +511,10 @@ func staticRoleContracts() []staticRoleContract {
{role: "k8s_query_admin", inherit: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
{role: "k8s_role_admin", login: true, createRole: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
{role: "k8s_reporting_owner", inherit: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
+ {role: "billing_meter_owner", inherit: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
{role: "k8s_metabase", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
{role: "cyclops_usage_reader", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
+ {role: "cyclops_meter_writer", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
}
}
@@ -532,6 +537,7 @@ func staticMembershipContracts(migrationOwner string) []staticMembershipContract
return []staticMembershipContract{
{role: "k8s_state_owner", member: migrationOwner, admin: false, inherit: false, set: true},
{role: "k8s_reporting_owner", member: migrationOwner, admin: false, inherit: false, set: true},
+ {role: "billing_meter_owner", member: migrationOwner, admin: false, inherit: false, set: true},
{role: "k8s_query_tenant", member: "k8s_role_admin", admin: true, inherit: false, set: false},
{role: "k8s_query_admin", member: "k8s_reporting_owner", admin: false, inherit: true, set: false},
}
@@ -705,8 +711,8 @@ func readReportingACLs(ctx context.Context, transaction pgx.Tx) ([]reportingACL,
routine.oid,
case
when namespace.nspname = 'k8s_reporting'
- and routine.proname = 'usage_sandbox_events'
- and routine.proargtypes = '25 1184 1184'::oidvector then 'usage_sandbox_events'
+ and routine.proname in ('usage_sandbox_events', 'reservation_hour_facts', 'reservation_meter_status')
+ and routine.proargtypes = '25 1184 1184'::oidvector then routine.proname
else routine.oid::regprocedure::text
end,
routine.proowner::regrole::text,
@@ -724,7 +730,7 @@ func readReportingACLs(ctx context.Context, transaction pgx.Tx) ([]reportingACL,
acl.grantee in ('k8s_reporting_owner'::regrole, 'k8s_metabase'::regrole, 'cyclops_usage_reader'::regrole)
or (
namespace.nspname = 'k8s_reporting'
- and routine.proname = 'usage_sandbox_events'
+ and routine.proname in ('usage_sandbox_events', 'reservation_hour_facts', 'reservation_meter_status')
and routine.proargtypes = '25 1184 1184'::oidvector
)
)
@@ -775,10 +781,15 @@ func expectedReportingACLs() []reportingACL {
{object: reportingObject{kind: reportingObjectSchema, schema: "k8s_state"}, owner: "k8s_state_owner", privilege: reportingPrivilegeUsage, grantee: "k8s_reporting_owner", grantor: "k8s_state_owner"},
{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_state", name: "resource_state"}, owner: "k8s_state_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_reporting_owner", grantor: "k8s_state_owner"},
{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_state", name: "resource_event_outbox"}, owner: "k8s_state_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_reporting_owner", grantor: "k8s_state_owner"},
+ {object: reportingObject{kind: reportingObjectSchema, schema: "billing_meter"}, owner: "billing_meter_owner", privilege: reportingPrivilegeUsage, grantee: "k8s_reporting_owner", grantor: "billing_meter_owner"},
+ {object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_current"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_reporting_owner", grantor: "billing_meter_owner"},
+ {object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_collection_current"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_reporting_owner", grantor: "billing_meter_owner"},
{object: reportingObject{kind: reportingObjectSchema, schema: "k8s_reporting"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeUsage, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},
{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_reporting", name: "current_resources"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},
{object: reportingObject{kind: reportingObjectSchema, schema: "k8s_reporting"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeUsage, grantee: "cyclops_usage_reader", grantor: "k8s_reporting_owner"},
{object: reportingObject{kind: reportingObjectRoutine}, routineIdentity: "usage_sandbox_events", owner: "k8s_reporting_owner", privilege: reportingPrivilegeExecute, grantee: "cyclops_usage_reader", grantor: "k8s_reporting_owner"},
+ {object: reportingObject{kind: reportingObjectRoutine}, routineIdentity: "reservation_hour_facts", owner: "k8s_reporting_owner", privilege: reportingPrivilegeExecute, grantee: "cyclops_usage_reader", grantor: "k8s_reporting_owner"},
+ {object: reportingObject{kind: reportingObjectRoutine}, routineIdentity: "reservation_meter_status", owner: "k8s_reporting_owner", privilege: reportingPrivilegeExecute, grantee: "cyclops_usage_reader", grantor: "k8s_reporting_owner"},
}
}
@@ -800,26 +811,28 @@ func containsReportingACL(acls []reportingACL, want reportingACL) bool {
}
func reconcileUsageRoutineExecute(ctx context.Context, transaction pgx.Tx) error {
- var routineOID uint32
- if err := transaction.QueryRow(ctx, `
- select routine.oid
- from pg_proc as routine
- join pg_namespace as namespace on namespace.oid = routine.pronamespace
- where namespace.nspname = 'k8s_reporting'
- and routine.proname = 'usage_sandbox_events'
- and routine.proargtypes = '25 1184 1184'::oidvector`).Scan(&routineOID); err != nil {
- return fmt.Errorf("resolve usage reporting routine: %w", err)
- }
- acl := reportingACL{
- object: reportingObject{kind: reportingObjectRoutine, routineOID: routineOID},
- routineIdentity: "usage_sandbox_events",
- owner: "k8s_reporting_owner",
- privilege: reportingPrivilegeExecute,
- grantee: "cyclops_usage_reader",
- grantor: "k8s_reporting_owner",
- }
- if err := newRuntimeDDL(transaction).grantReportingACL(ctx, acl); err != nil {
- return fmt.Errorf("reconcile usage routine execute grant: %w", err)
+ for _, routineIdentity := range []string{"usage_sandbox_events", "reservation_hour_facts", "reservation_meter_status"} {
+ var routineOID uint32
+ if err := transaction.QueryRow(ctx, `
+ select routine.oid
+ from pg_proc as routine
+ join pg_namespace as namespace on namespace.oid = routine.pronamespace
+ where namespace.nspname = 'k8s_reporting'
+ and routine.proname = $1
+ and routine.proargtypes = '25 1184 1184'::oidvector`, routineIdentity).Scan(&routineOID); err != nil {
+ return fmt.Errorf("resolve usage reporting routine %s: %w", routineIdentity, err)
+ }
+ acl := reportingACL{
+ object: reportingObject{kind: reportingObjectRoutine, routineOID: routineOID},
+ routineIdentity: routineIdentity,
+ owner: "k8s_reporting_owner",
+ privilege: reportingPrivilegeExecute,
+ grantee: "cyclops_usage_reader",
+ grantor: "k8s_reporting_owner",
+ }
+ if err := newRuntimeDDL(transaction).grantReportingACL(ctx, acl); err != nil {
+ return fmt.Errorf("reconcile usage routine %s execute grant: %w", routineIdentity, err)
+ }
}
return nil
}
diff --git a/libs/fleet/backend/database/migrator_pg_test.go b/libs/fleet/backend/database/migrator_pg_test.go
index 3f18a9d148..a43fd59ccf 100644
--- a/libs/fleet/backend/database/migrator_pg_test.go
+++ b/libs/fleet/backend/database/migrator_pg_test.go
@@ -24,6 +24,7 @@ const tenantCredentialFingerprintLookup = `select credential_fingerprint from k8
var staticMigrationRoles = []string{
"cyclops_app",
"cyclops_usage_reader",
+ "cyclops_meter_writer",
"k8s_state_owner",
"k8s_state_writer",
"k8s_state_exporter",
@@ -31,6 +32,7 @@ var staticMigrationRoles = []string{
"k8s_query_admin",
"k8s_role_admin",
"k8s_reporting_owner",
+ "billing_meter_owner",
"k8s_metabase",
}
@@ -1501,6 +1503,7 @@ func testCredentialURLs(t *testing.T, adminURL string) CredentialURLs {
RoleAdmin: withRole("k8s_role_admin"),
Metabase: withRole("k8s_metabase"),
Usage: withRole("cyclops_usage_reader"),
+ Meter: withRole("cyclops_meter_writer"),
}
}
@@ -1678,8 +1681,10 @@ func assertRoleContract(t *testing.T, ctx context.Context, adminURL string) {
"k8s_query_admin": {false, true, false},
"k8s_role_admin": {true, false, true},
"k8s_reporting_owner": {false, true, false},
+ "billing_meter_owner": {false, true, false},
"k8s_metabase": {true, false, false},
"cyclops_usage_reader": {true, false, false},
+ "cyclops_meter_writer": {true, false, false},
}
for role, want := range expected {
var login, inherit, createRole, super, createDB, replication, bypassRLS bool
@@ -2310,10 +2315,15 @@ func assertExactReportingACLContract(t *testing.T, ctx context.Context, connecti
('schema:k8s_state', 'USAGE', 'k8s_reporting_owner', 'k8s_state_owner', false),
('relation:k8s_state.resource_state', 'SELECT', 'k8s_reporting_owner', 'k8s_state_owner', false),
('relation:k8s_state.resource_event_outbox', 'SELECT', 'k8s_reporting_owner', 'k8s_state_owner', false),
+ ('schema:billing_meter', 'USAGE', 'k8s_reporting_owner', 'billing_meter_owner', false),
+ ('relation:billing_meter.reservation_hour_current', 'SELECT', 'k8s_reporting_owner', 'billing_meter_owner', false),
+ ('relation:billing_meter.reservation_hour_collection_current', 'SELECT', 'k8s_reporting_owner', 'billing_meter_owner', false),
('schema:k8s_reporting', 'USAGE', 'k8s_metabase', 'k8s_reporting_owner', false),
('relation:k8s_reporting.current_resources', 'SELECT', 'k8s_metabase', 'k8s_reporting_owner', false),
('schema:k8s_reporting', 'USAGE', 'cyclops_usage_reader', 'k8s_reporting_owner', false),
- ('routine:k8s_reporting.usage_sandbox_events(text,timestamp with time zone,timestamp with time zone)', 'EXECUTE', 'cyclops_usage_reader', 'k8s_reporting_owner', false)
+ ('routine:k8s_reporting.usage_sandbox_events(text,timestamp with time zone,timestamp with time zone)', 'EXECUTE', 'cyclops_usage_reader', 'k8s_reporting_owner', false),
+ ('routine:k8s_reporting.reservation_hour_facts(text,timestamp with time zone,timestamp with time zone)', 'EXECUTE', 'cyclops_usage_reader', 'k8s_reporting_owner', false),
+ ('routine:k8s_reporting.reservation_meter_status(text,timestamp with time zone,timestamp with time zone)', 'EXECUTE', 'cyclops_usage_reader', 'k8s_reporting_owner', false)
)
select count(*) from (
(select * from actual except select * from expected)
@@ -2349,7 +2359,7 @@ func assertExactReportingACLContract(t *testing.T, ctx context.Context, connecti
join pg_namespace as namespace on namespace.oid = routine.pronamespace
join lateral aclexplode(routine.proacl) as acl on true
where namespace.nspname = 'k8s_reporting'
- and routine.proname = 'usage_sandbox_events'
+ and routine.proname in ('usage_sandbox_events', 'reservation_hour_facts', 'reservation_meter_status')
and routine.proargtypes = '25 1184 1184'::oidvector
and acl.grantee <> routine.proowner
and (acl.grantee <> 'cyclops_usage_reader'::regrole or acl.privilege_type <> 'EXECUTE' or acl.is_grantable)
diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.go
index ff3973dd0e..fac1cf362d 100644
--- a/libs/fleet/backend/database/migrator_test.go
+++ b/libs/fleet/backend/database/migrator_test.go
@@ -520,8 +520,8 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if len(files) != 4 {
- t.Fatalf("expected exactly four migrations, got %d", len(files))
+ if len(files) != 5 {
+ t.Fatalf("expected exactly five migrations, got %d", len(files))
}
initial := files[0]
if initial.Version != 1 || initial.Name != "000001_initial_schema.sql" {
@@ -609,6 +609,26 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {
t.Errorf("usage filter migration is missing contract %q", expected)
}
}
+
+ meter := files[4]
+ if meter.Version != 5 || meter.Name != "000005_hourly_reservation_meter.sql" {
+ t.Fatalf("expected version 5 reservation meter migration, got version=%d name=%q", meter.Version, meter.Name)
+ }
+ for _, expected := range []string{
+ "CREATE ROLE billing_meter_owner NOLOGIN",
+ "CREATE ROLE cyclops_meter_writer LOGIN",
+ "CREATE TABLE billing_meter.reservation_hour_fact",
+ "CREATE VIEW billing_meter.reservation_hour_current",
+ "BEFORE UPDATE OR DELETE OR TRUNCATE ON billing_meter.reservation_hour_fact",
+ "CREATE FUNCTION k8s_api.sandbox_meter_tenant",
+ "CREATE FUNCTION k8s_reporting.reservation_hour_facts",
+ "GRANT SELECT, INSERT ON TABLE billing_meter.reservation_hour_collection, billing_meter.reservation_hour_fact TO cyclops_meter_writer",
+ "GRANT EXECUTE ON FUNCTION k8s_reporting.reservation_hour_facts(text, timestamptz, timestamptz) TO cyclops_usage_reader",
+ } {
+ if !strings.Contains(meter.SQL, expected) {
+ t.Errorf("reservation meter migration is missing contract %q", expected)
+ }
+ }
}
func TestAppliedMigrationLedgerUsesApplicationOrder(t *testing.T) {
@@ -726,12 +746,13 @@ func TestCredentialURLsExcludeDynamicTenantRoles(t *testing.T) {
RoleAdmin: "postgres://k8s_role_admin:pw@db/cyclops",
Metabase: "postgres://k8s_metabase:pw@db/cyclops",
Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",
+ Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",
})
if err != nil {
t.Fatalf("parse fixed runtime credentials: %v", err)
}
- if len(credentials) != 6 {
- t.Fatalf("credential count = %d, want 6", len(credentials))
+ if len(credentials) != 7 {
+ t.Fatalf("credential count = %d, want 7", len(credentials))
}
for _, credential := range credentials {
if credential.Role == "k8s_query_broker" || strings.HasPrefix(credential.Role, "k8s_tenant_") {
@@ -755,8 +776,10 @@ func TestStaticRoleContractsAreAllowlistedAndFixed(t *testing.T) {
"k8s_query_admin": {false, true, false, false, -1, staticRoleValidUntilInfinity},
"k8s_role_admin": {true, false, true, false, -1, staticRoleValidUntilInfinity},
"k8s_reporting_owner": {false, true, false, false, -1, staticRoleValidUntilInfinity},
+ "billing_meter_owner": {false, true, false, false, -1, staticRoleValidUntilInfinity},
"k8s_metabase": {true, false, false, false, -1, staticRoleValidUntilInfinity},
"cyclops_usage_reader": {true, false, false, false, -1, staticRoleValidUntilInfinity},
+ "cyclops_meter_writer": {true, false, false, false, -1, staticRoleValidUntilInfinity},
}
if len(contracts) != len(want) {
t.Fatalf("static role contract count = %d, want %d", len(contracts), len(want))
@@ -781,6 +804,7 @@ func TestStaticMembershipContractsUsePG16Options(t *testing.T) {
want := []staticMembershipContract{
{role: "k8s_state_owner", member: "migration_owner", admin: false, inherit: false, set: true},
{role: "k8s_reporting_owner", member: "migration_owner", admin: false, inherit: false, set: true},
+ {role: "billing_meter_owner", member: "migration_owner", admin: false, inherit: false, set: true},
{role: "k8s_query_tenant", member: "k8s_role_admin", admin: true, inherit: false, set: false},
{role: "k8s_query_admin", member: "k8s_reporting_owner", admin: false, inherit: true, set: false},
}
@@ -805,6 +829,7 @@ func TestCredentialURLsRequireExpectedRoleNames(t *testing.T) {
RoleAdmin: "postgres://k8s_role_admin:pw@db/cyclops",
Metabase: "postgres://k8s_metabase:pw@db/cyclops",
Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",
+ Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",
})
if err == nil {
t.Fatal("expected application role-name validation")
@@ -822,6 +847,7 @@ func TestCredentialURLsRejectEmptyPassword(t *testing.T) {
RoleAdmin: "postgres://k8s_role_admin:pw@db/cyclops",
Metabase: "postgres://k8s_metabase:pw@db/cyclops",
Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",
+ Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",
})
if err == nil || err.Error() != "application credential database URL must include a password" {
t.Fatalf("expected empty password rejection, got %v", err)
@@ -836,6 +862,7 @@ func TestCredentialURLParseErrorsPreserveCauseForBoundaryClassification(t *testi
RoleAdmin: "postgres://k8s_role_admin:pw@db/cyclops",
Metabase: "postgres://k8s_metabase:pw@db/cyclops",
Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",
+ Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",
})
if !errors.Is(err, ErrInvalidConfiguration) {
t.Fatalf("error = %v, want ErrInvalidConfiguration", err)
diff --git a/libs/fleet/backend/database/test-setup.sh b/libs/fleet/backend/database/test-setup.sh
index 7390be1555..10886bd802 100755
--- a/libs/fleet/backend/database/test-setup.sh
+++ b/libs/fleet/backend/database/test-setup.sh
@@ -22,8 +22,10 @@ static_roles=(
k8s_query_admin
k8s_role_admin
k8s_reporting_owner
+ billing_meter_owner
k8s_metabase
cyclops_usage_reader
+ cyclops_meter_writer
)
base_url="${admin_url%%\?*}"
@@ -84,6 +86,7 @@ exporter_url="$(url_for k8s_state_exporter state-exporter-password)"
role_admin_url="$(url_for k8s_role_admin role-admin-password)"
metabase_url="$(url_for k8s_metabase metabase-password)"
usage_url="$(url_for cyclops_usage_reader usage-reader-password)"
+meter_url="$(url_for cyclops_meter_writer meter-writer-password)"
env_file="${CYCLOPS_TEST_DATABASE_ENV_FILE:-${GITHUB_ENV:-}}"
if [[ -z "$env_file" ]]; then
@@ -153,6 +156,7 @@ STATE_EXPORTER_DATABASE_URL="$exporter_url" \
STATE_ROLE_ADMIN_DATABASE_URL="$role_admin_url" \
METABASE_DATABASE_URL="$metabase_url" \
USAGE_DATABASE_URL="$usage_url" \
+METER_DATABASE_URL="$meter_url" \
go run ./cmd/db-migrate
cat >> "$env_file" <<EOF_ENV
diff --git a/libs/fleet/backend/main.go b/libs/fleet/backend/main.go
index 6240db0409..99b827ce5a 100644
--- a/libs/fleet/backend/main.go
+++ b/libs/fleet/backend/main.go
@@ -614,5 +614,5 @@ func initializeUsageProvider(ctx context.Context, cfg config.UsageConfiguration)
events.Close()
return nil, noop, fmt.Errorf("create DataFusion allocation client: %w", err)
}
- return usage.NewProvider(events, allocations, nil), events.Close, nil
+ return usage.NewProviderWithReservations(events, allocations, events, cfg.QueryCluster, nil), events.Close, nil
}
diff --git a/libs/fleet/backend/metering/collector.go b/libs/fleet/backend/metering/collector.go
new file mode 100644
index 0000000000..de09398974
--- /dev/null
+++ b/libs/fleet/backend/metering/collector.go
@@ -0,0 +1,334 @@
+package metering
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "math"
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+)
+
+const (
+ CPUQuery = "cua_osgym_sandbox_virtual_cpu_cores"
+ MemoryQuery = "cua_osgym_sandbox_virtual_memory_bytes"
+ ReadyQuery = "cua_osgym_sandbox_ready"
+ SourceQuery = `up{job="kube-state-metrics-required"}`
+)
+
+type RangeQuerier interface {
+ QueryRange(context.Context, string, time.Time, time.Time, time.Duration) ([]Series, error)
+}
+
+type TenantResolver interface {
+ ResolveTenant(context.Context, string, string, string) (string, error)
+}
+
+type FactWriter interface {
+ AppendFact(context.Context, HourFact) (bool, error)
+ CompleteHour(context.Context, HourCompletion) (bool, error)
+}
+
+type HourFact struct {
+ FactID uuid.UUID
+ LogicalKey string
+ ClusterID string
+ CapsuleTenant string
+ Namespace string
+ SandboxUID string
+ SandboxName string
+ PoolName string
+ Runtime string
+ HourStart time.Time
+ HourEnd time.Time
+ VirtualCPUCoreSeconds float64
+ VirtualMemoryByteSeconds float64
+ ReadySeconds float64
+ CoveredSeconds float64
+ ScrapeIntervalSeconds int
+ SourceSHA256 string
+ CollectionRunID uuid.UUID
+}
+
+type HourCompletion struct {
+ CollectionRunID uuid.UUID
+ LogicalKey string
+ ClusterID string
+ HourStart time.Time
+ HourEnd time.Time
+ CoveredSeconds float64
+ DiscoveredSandboxes int
+ InsertedFacts int
+ UnchangedFacts int
+ SourceSHA256 string
+}
+
+type Collector struct {
+ Prometheus RangeQuerier
+ Tenants TenantResolver
+ Writer FactWriter
+ ClusterID string
+ Step time.Duration
+ MaxSampleValidity time.Duration
+ MinimumCoverage time.Duration
+}
+
+type CollectResult struct {
+ Discovered int
+ Inserted int
+ Unchanged int
+ Coverage time.Duration
+}
+
+type sandboxIdentity struct {
+ Namespace string
+ SandboxUID string
+ SandboxName string
+ PoolName string
+ Runtime string
+}
+
+type sandboxSamples struct {
+ identity sandboxIdentity
+ cpu []Sample
+ memory []Sample
+ ready []Sample
+}
+
+func (c Collector) CollectHour(ctx context.Context, hourStart time.Time) (CollectResult, error) {
+ if c.Prometheus == nil || c.Tenants == nil || c.Writer == nil || strings.TrimSpace(c.ClusterID) == "" {
+ return CollectResult{}, fmt.Errorf("meter collector is not configured")
+ }
+ if c.Step <= 0 || c.MaxSampleValidity <= 0 || c.MinimumCoverage <= 0 {
+ return CollectResult{}, fmt.Errorf("meter collector timing is invalid")
+ }
+ hourStart = hourStart.UTC()
+ if !hourStart.Equal(hourStart.Truncate(time.Hour)) {
+ return CollectResult{}, fmt.Errorf("meter hour must start on a UTC hour")
+ }
+ hourEnd := hourStart.Add(time.Hour)
+ queryStart := hourStart.Add(-c.MaxSampleValidity)
+
+ sourceSeries, err := c.Prometheus.QueryRange(ctx, SourceQuery, queryStart, hourEnd, c.Step)
+ if err != nil {
+ return CollectResult{}, fmt.Errorf("query KSM source coverage: %w", err)
+ }
+ if len(sourceSeries) != 1 {
+ return CollectResult{}, fmt.Errorf("KSM source coverage returned %d series, want 1", len(sourceSeries))
+ }
+ sourceIntegral, err := Integrate(sourceSeries[0].Samples, hourStart, hourEnd, c.MaxSampleValidity)
+ if err != nil {
+ return CollectResult{}, fmt.Errorf("integrate KSM source coverage: %w", err)
+ }
+ coverage := time.Duration(sourceIntegral.ValueSeconds * float64(time.Second))
+ if coverage < c.MinimumCoverage {
+ return CollectResult{}, fmt.Errorf("KSM source coverage %s is below required %s", coverage, c.MinimumCoverage)
+ }
+
+ metricSeries := make(map[string][]Series, 3)
+ for _, query := range []string{CPUQuery, MemoryQuery, ReadyQuery} {
+ series, queryErr := c.Prometheus.QueryRange(ctx, query, queryStart, hourEnd, c.Step)
+ if queryErr != nil {
+ return CollectResult{}, fmt.Errorf("query %s: %w", query, queryErr)
+ }
+ metricSeries[query] = series
+ }
+
+ sandboxes, err := mergeSandboxSeries(metricSeries)
+ if err != nil {
+ return CollectResult{}, err
+ }
+ result := CollectResult{Discovered: len(sandboxes), Coverage: coverage}
+ collectionRunID := uuid.New()
+ factHashes := make([]string, 0, len(sandboxes))
+ keys := make([]string, 0, len(sandboxes))
+ for key := range sandboxes {
+ keys = append(keys, key)
+ }
+ sort.Strings(keys)
+
+ for _, key := range keys {
+ sandbox := sandboxes[key]
+ cpu, err := Integrate(sandbox.cpu, hourStart, hourEnd, c.MaxSampleValidity)
+ if err != nil {
+ return CollectResult{}, fmt.Errorf("integrate CPU for sandbox %s: %w", sandbox.identity.SandboxUID, err)
+ }
+ memory, err := Integrate(sandbox.memory, hourStart, hourEnd, c.MaxSampleValidity)
+ if err != nil {
+ return CollectResult{}, fmt.Errorf("integrate memory for sandbox %s: %w", sandbox.identity.SandboxUID, err)
+ }
+ if cpu.ValueSeconds < 0 || memory.ValueSeconds < 0 {
+ return CollectResult{}, fmt.Errorf("sandbox %s has negative reservation samples", sandbox.identity.SandboxUID)
+ }
+ if math.Abs(cpu.PresentSeconds-memory.PresentSeconds) > c.Step.Seconds() {
+ return CollectResult{}, fmt.Errorf("sandbox %s CPU and memory sample coverage diverged", sandbox.identity.SandboxUID)
+ }
+ ready, err := Integrate(sandbox.ready, hourStart, hourEnd, c.MaxSampleValidity)
+ if err != nil {
+ return CollectResult{}, fmt.Errorf("integrate readiness for sandbox %s: %w", sandbox.identity.SandboxUID, err)
+ }
+ tenant, err := c.Tenants.ResolveTenant(ctx, c.ClusterID, sandbox.identity.Namespace, sandbox.identity.SandboxUID)
+ if err != nil {
+ return CollectResult{}, fmt.Errorf("resolve tenant for sandbox %s: %w", sandbox.identity.SandboxUID, err)
+ }
+ fact := HourFact{
+ FactID: uuid.New(),
+ LogicalKey: c.ClusterID + "/" + sandbox.identity.SandboxUID + "/" + hourStart.Format(time.RFC3339),
+ ClusterID: c.ClusterID,
+ CapsuleTenant: tenant,
+ Namespace: sandbox.identity.Namespace,
+ SandboxUID: sandbox.identity.SandboxUID,
+ SandboxName: sandbox.identity.SandboxName,
+ PoolName: sandbox.identity.PoolName,
+ Runtime: sandbox.identity.Runtime,
+ HourStart: hourStart,
+ HourEnd: hourEnd,
+ VirtualCPUCoreSeconds: cpu.ValueSeconds,
+ VirtualMemoryByteSeconds: memory.ValueSeconds,
+ ReadySeconds: ready.ValueSeconds,
+ CoveredSeconds: sourceIntegral.ValueSeconds,
+ ScrapeIntervalSeconds: int(c.Step.Seconds()),
+ CollectionRunID: collectionRunID,
+ }
+ fact.SourceSHA256 = hashFactSource(fact, sandbox, sourceSeries[0].Samples)
+ factHashes = append(factHashes, fact.SourceSHA256)
+ inserted, err := c.Writer.AppendFact(ctx, fact)
+ if err != nil {
+ return CollectResult{}, fmt.Errorf("append sandbox %s reservation fact: %w", sandbox.identity.SandboxUID, err)
+ }
+ if inserted {
+ result.Inserted++
+ } else {
+ result.Unchanged++
+ }
+ }
+ sort.Strings(factHashes)
+ completion := HourCompletion{
+ CollectionRunID: collectionRunID,
+ LogicalKey: c.ClusterID + "/" + hourStart.Format(time.RFC3339),
+ ClusterID: c.ClusterID,
+ HourStart: hourStart,
+ HourEnd: hourEnd,
+ CoveredSeconds: sourceIntegral.ValueSeconds,
+ DiscoveredSandboxes: result.Discovered,
+ InsertedFacts: result.Inserted,
+ UnchangedFacts: result.Unchanged,
+ SourceSHA256: hashCollectionSource(c.ClusterID, hourStart, sourceSeries[0].Samples, factHashes),
+ }
+ if _, err := c.Writer.CompleteHour(ctx, completion); err != nil {
+ return CollectResult{}, fmt.Errorf("complete reservation hour: %w", err)
+ }
+ return result, nil
+}
+
+func mergeSandboxSeries(metrics map[string][]Series) (map[string]*sandboxSamples, error) {
+ out := make(map[string]*sandboxSamples)
+ for _, metric := range []string{CPUQuery, MemoryQuery, ReadyQuery} {
+ for _, series := range metrics[metric] {
+ identity, err := identityFromLabels(series.Labels)
+ if err != nil {
+ return nil, fmt.Errorf("invalid %s labels: %w", metric, err)
+ }
+ key := identity.Namespace + "/" + identity.SandboxUID
+ sandbox := out[key]
+ if sandbox == nil {
+ sandbox = &sandboxSamples{identity: identity}
+ out[key] = sandbox
+ } else if sandbox.identity != identity {
+ return nil, fmt.Errorf("sandbox %s metric labels disagree", identity.SandboxUID)
+ }
+ switch metric {
+ case CPUQuery:
+ if sandbox.cpu != nil {
+ return nil, fmt.Errorf("sandbox %s has duplicate CPU series", identity.SandboxUID)
+ }
+ sandbox.cpu = series.Samples
+ case MemoryQuery:
+ if sandbox.memory != nil {
+ return nil, fmt.Errorf("sandbox %s has duplicate memory series", identity.SandboxUID)
+ }
+ sandbox.memory = series.Samples
+ case ReadyQuery:
+ if sandbox.ready != nil {
+ return nil, fmt.Errorf("sandbox %s has duplicate ready series", identity.SandboxUID)
+ }
+ sandbox.ready = series.Samples
+ }
+ }
+ }
+ for _, sandbox := range out {
+ if sandbox.cpu == nil || sandbox.memory == nil {
+ return nil, fmt.Errorf("sandbox %s is missing CPU or memory series", sandbox.identity.SandboxUID)
+ }
+ }
+ return out, nil
+}
+
+func identityFromLabels(labels map[string]string) (sandboxIdentity, error) {
+ poolName := strings.TrimSpace(labels["pool"])
+ if poolName == "" {
+ poolName = strings.TrimSpace(labels["warmpool"])
+ }
+ identity := sandboxIdentity{
+ Namespace: strings.TrimSpace(labels["namespace"]),
+ SandboxUID: strings.TrimSpace(labels["sandbox_uid"]),
+ SandboxName: strings.TrimSpace(labels["sandbox"]),
+ PoolName: poolName,
+ Runtime: strings.TrimSpace(labels["runtime"]),
+ }
+ if identity.Namespace == "" || identity.SandboxUID == "" || identity.SandboxName == "" || identity.PoolName == "" || identity.Runtime == "" {
+ return sandboxIdentity{}, fmt.Errorf("required identity label is empty")
+ }
+ return identity, nil
+}
+
+func hashFactSource(fact HourFact, sandbox *sandboxSamples, source []Sample) string {
+ hash := sha256.New()
+ for _, value := range []string{
+ fact.LogicalKey,
+ fact.Namespace,
+ fact.SandboxName,
+ fact.PoolName,
+ fact.Runtime,
+ strconv.Itoa(fact.ScrapeIntervalSeconds),
+ } {
+ _, _ = hash.Write([]byte(value))
+ _, _ = hash.Write([]byte{0})
+ }
+ for _, samples := range [][]Sample{sandbox.cpu, sandbox.memory, sandbox.ready, source} {
+ ordered := append([]Sample(nil), samples...)
+ sort.Slice(ordered, func(i, j int) bool { return ordered[i].Timestamp.Before(ordered[j].Timestamp) })
+ for _, sample := range ordered {
+ _, _ = hash.Write([]byte(strconv.FormatInt(sample.Timestamp.UnixNano(), 10)))
+ _, _ = hash.Write([]byte{'='})
+ _, _ = hash.Write([]byte(strconv.FormatFloat(sample.Value, 'g', -1, 64)))
+ _, _ = hash.Write([]byte{0})
+ }
+ _, _ = hash.Write([]byte{0xff})
+ }
+ return hex.EncodeToString(hash.Sum(nil))
+}
+
+func hashCollectionSource(clusterID string, hourStart time.Time, source []Sample, factHashes []string) string {
+ hash := sha256.New()
+ _, _ = hash.Write([]byte(clusterID))
+ _, _ = hash.Write([]byte{0})
+ _, _ = hash.Write([]byte(hourStart.Format(time.RFC3339)))
+ _, _ = hash.Write([]byte{0})
+ for _, sample := range source {
+ _, _ = hash.Write([]byte(strconv.FormatInt(sample.Timestamp.UnixNano(), 10)))
+ _, _ = hash.Write([]byte{'='})
+ _, _ = hash.Write([]byte(strconv.FormatFloat(sample.Value, 'g', -1, 64)))
+ _, _ = hash.Write([]byte{0})
+ }
+ for _, factHash := range factHashes {
+ _, _ = hash.Write([]byte(factHash))
+ _, _ = hash.Write([]byte{0})
+ }
+ return hex.EncodeToString(hash.Sum(nil))
+}
diff --git a/libs/fleet/backend/metering/collector_test.go b/libs/fleet/backend/metering/collector_test.go
new file mode 100644
index 0000000000..2c835a7100
--- /dev/null
+++ b/libs/fleet/backend/metering/collector_test.go
@@ -0,0 +1,113 @@
+package metering
+
+import (
+ "context"
+ "strings"
+ "testing"
+ "time"
+)
+
+type fakeQuerier map[string][]Series
+
+func (f fakeQuerier) QueryRange(_ context.Context, query string, _, _ time.Time, _ time.Duration) ([]Series, error) {
+ return f[query], nil
+}
+
+type fakeTenantResolver struct{ tenant string }
+
+func (f fakeTenantResolver) ResolveTenant(context.Context, string, string, string) (string, error) {
+ return f.tenant, nil
+}
+
+type fakeFactWriter struct{ facts []HourFact }
+
+func (f *fakeFactWriter) AppendFact(_ context.Context, fact HourFact) (bool, error) {
+ f.facts = append(f.facts, fact)
+ return true, nil
+}
+
+func (f *fakeFactWriter) CompleteHour(context.Context, HourCompletion) (bool, error) {
+ return true, nil
+}
+
+func TestCollectorMaterializesReservationHour(t *testing.T) {
+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)
+ labels := map[string]string{
+ "namespace": "tenant-a",
+ "sandbox": "sandbox-a",
+ "sandbox_uid": "uid-a",
+ "pool": "pool-a",
+ "runtime": "kubevirt",
+ }
+ writer := &fakeFactWriter{}
+ collector := Collector{
+ Prometheus: fakeQuerier{
+ SourceQuery: {{Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, 1)}},
+ CPUQuery: {{Labels: labels, Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, 4)}},
+ MemoryQuery: {{Labels: labels, Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, float64(8<<30))}},
+ ReadyQuery: {{Labels: labels, Samples: constantSamples(start.Add(-15*time.Second), start.Add(time.Hour), 15*time.Second, 1)}},
+ },
+ Tenants: fakeTenantResolver{tenant: "personal:user-a"},
+ Writer: writer,
+ ClusterID: "kopf-k3s",
+ Step: 15 * time.Second,
+ MaxSampleValidity: time.Minute,
+ MinimumCoverage: 3570 * time.Second,
+ }
+
+ result, err := collector.CollectHour(context.Background(), start)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.Discovered != 1 || result.Inserted != 1 || len(writer.facts) != 1 {
+ t.Fatalf("result = %+v, facts = %d", result, len(writer.facts))
+ }
+ fact := writer.facts[0]
+ if fact.VirtualCPUCoreSeconds != 4*3600 || fact.VirtualMemoryByteSeconds != float64(8<<30)*3600 || fact.ReadySeconds != 3600 || fact.CoveredSeconds != 3600 {
+ t.Fatalf("fact quantities = %+v", fact)
+ }
+ if fact.CapsuleTenant != "personal:user-a" || fact.SourceSHA256 == "" || !strings.Contains(fact.LogicalKey, "uid-a") {
+ t.Fatalf("fact identity = %+v", fact)
+ }
+}
+
+func TestCollectorFailsClosedOnSourceCoverage(t *testing.T) {
+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)
+ collector := Collector{
+ Prometheus: fakeQuerier{SourceQuery: {{Samples: []Sample{{Timestamp: start, Value: 1}}}}},
+ Tenants: fakeTenantResolver{tenant: "personal:user-a"},
+ Writer: &fakeFactWriter{},
+ ClusterID: "kopf-k3s",
+ Step: 15 * time.Second,
+ MaxSampleValidity: time.Minute,
+ MinimumCoverage: 3570 * time.Second,
+ }
+ _, err := collector.CollectHour(context.Background(), start)
+ if err == nil || !strings.Contains(err.Error(), "below required") {
+ t.Fatalf("error = %v, want coverage rejection", err)
+ }
+}
+
+func TestIdentityFromLabelsFallsBackToWarmPoolLabel(t *testing.T) {
+ identity, err := identityFromLabels(map[string]string{
+ "namespace": "tenant-a",
+ "sandbox": "sandbox-a",
+ "sandbox_uid": "uid-a",
+ "warmpool": "pool-a",
+ "runtime": "macos",
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ if identity.PoolName != "pool-a" {
+ t.Fatalf("pool = %q, want pool-a", identity.PoolName)
+ }
+}
+
+func constantSamples(start, end time.Time, step time.Duration, value float64) []Sample {
+ var samples []Sample
+ for timestamp := start; !timestamp.After(end); timestamp = timestamp.Add(step) {
+ samples = append(samples, Sample{Timestamp: timestamp, Value: value})
+ }
+ return samples
+}
diff --git a/libs/fleet/backend/metering/integrate.go b/libs/fleet/backend/metering/integrate.go
new file mode 100644
index 0000000000..3e0695f3e2
--- /dev/null
+++ b/libs/fleet/backend/metering/integrate.go
@@ -0,0 +1,57 @@
+package metering
+
+import (
+ "fmt"
+ "math"
+ "sort"
+ "time"
+)
+
+type Sample struct {
+ Timestamp time.Time
+ Value float64
+}
+
+type Integral struct {
+ ValueSeconds float64
+ PresentSeconds float64
+}
+
+func Integrate(samples []Sample, start, end time.Time, maxValidity time.Duration) (Integral, error) {
+ if start.IsZero() || end.IsZero() || !end.After(start) || maxValidity <= 0 {
+ return Integral{}, fmt.Errorf("invalid integration window")
+ }
+
+ ordered := append([]Sample(nil), samples...)
+ sort.Slice(ordered, func(i, j int) bool { return ordered[i].Timestamp.Before(ordered[j].Timestamp) })
+
+ var result Integral
+ for index, sample := range ordered {
+ if sample.Timestamp.IsZero() || math.IsNaN(sample.Value) || math.IsInf(sample.Value, 0) {
+ return Integral{}, fmt.Errorf("invalid metric sample")
+ }
+ if index > 0 && !sample.Timestamp.After(ordered[index-1].Timestamp) {
+ return Integral{}, fmt.Errorf("metric samples must have unique timestamps")
+ }
+
+ intervalStart := sample.Timestamp
+ if intervalStart.Before(start) {
+ intervalStart = start
+ }
+ intervalEnd := sample.Timestamp.Add(maxValidity)
+ if index+1 < len(ordered) && ordered[index+1].Timestamp.Before(intervalEnd) {
+ intervalEnd = ordered[index+1].Timestamp
+ }
+ if intervalEnd.After(end) {
+ intervalEnd = end
+ }
+ if !intervalEnd.After(intervalStart) || !intervalStart.Before(end) {
+ continue
+ }
+
+ seconds := intervalEnd.Sub(intervalStart).Seconds()
+ result.ValueSeconds += sample.Value * seconds
+ result.PresentSeconds += seconds
+ }
+ return result, nil
+}
diff --git a/libs/fleet/backend/metering/integrate_test.go b/libs/fleet/backend/metering/integrate_test.go
new file mode 100644
index 0000000000..201ef5164d
--- /dev/null
+++ b/libs/fleet/backend/metering/integrate_test.go
@@ -0,0 +1,40 @@
+package metering
+
+import (
+ "math"
+ "testing"
+ "time"
+)
+
+func TestIntegrateUsesPreviousSampleAtWindowBoundary(t *testing.T) {
+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)
+ result, err := Integrate([]Sample{
+ {Timestamp: start.Add(-15 * time.Second), Value: 4},
+ {Timestamp: start.Add(15 * time.Second), Value: 4},
+ }, start, start.Add(30*time.Second), time.Minute)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.ValueSeconds != 120 || result.PresentSeconds != 30 {
+ t.Fatalf("integral = %+v, want 120 value-seconds over 30 seconds", result)
+ }
+}
+
+func TestIntegrateCapsStaleSamples(t *testing.T) {
+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)
+ result, err := Integrate([]Sample{{Timestamp: start, Value: 8}}, start, start.Add(time.Hour), time.Minute)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if result.ValueSeconds != 480 || result.PresentSeconds != 60 {
+ t.Fatalf("integral = %+v, want one minute of validity", result)
+ }
+}
+
+func TestIntegrateRejectsNonFiniteSamples(t *testing.T) {
+ start := time.Date(2026, 8, 22, 7, 0, 0, 0, time.UTC)
+ _, err := Integrate([]Sample{{Timestamp: start, Value: math.NaN()}}, start, start.Add(time.Hour), time.Minute)
+ if err == nil {
+ t.Fatal("expected non-finite sample rejection")
+ }
+}
diff --git a/libs/fleet/backend/metering/postgres.go b/libs/fleet/backend/metering/postgres.go
new file mode 100644
index 0000000000..47c31a29ab
--- /dev/null
+++ b/libs/fleet/backend/metering/postgres.go
@@ -0,0 +1,189 @@
+package metering
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "time"
+
+ "github.com/google/uuid"
+ "github.com/jackc/pgx/v5"
+ "github.com/jackc/pgx/v5/pgxpool"
+)
+
+type PostgresStore struct {
+ pool *pgxpool.Pool
+}
+
+func NewPostgresStore(ctx context.Context, databaseURL string) (*PostgresStore, error) {
+ config, err := pgxpool.ParseConfig(databaseURL)
+ if err != nil {
+ return nil, fmt.Errorf("parse meter database URL: %w", err)
+ }
+ if config.ConnConfig.User != "cyclops_meter_writer" {
+ return nil, fmt.Errorf("meter database URL must use cyclops_meter_writer")
+ }
+ config.MaxConns = 2
+ config.MinConns = 0
+ config.MaxConnIdleTime = time.Minute
+ config.MaxConnLifetime = 15 * time.Minute
+ pool, err := pgxpool.NewWithConfig(ctx, config)
+ if err != nil {
+ return nil, fmt.Errorf("open meter database pool: %w", err)
+ }
+ return &PostgresStore{pool: pool}, nil
+}
+
+func (s *PostgresStore) Close() {
+ if s != nil && s.pool != nil {
+ s.pool.Close()
+ }
+}
+
+func (s *PostgresStore) ResolveTenant(ctx context.Context, clusterID, namespace, sandboxUID string) (string, error) {
+ var tenant string
+ if err := s.pool.QueryRow(ctx, `select k8s_api.sandbox_meter_tenant($1, $2, $3)`, clusterID, namespace, sandboxUID).Scan(&tenant); err != nil {
+ return "", fmt.Errorf("query sandbox tenant: %w", err)
+ }
+ if strings.TrimSpace(tenant) == "" {
+ return "", fmt.Errorf("sandbox tenant is empty")
+ }
+ return tenant, nil
+}
+
+func (s *PostgresStore) AppendFact(ctx context.Context, fact HourFact) (bool, error) {
+ if err := validateHourFact(fact); err != nil {
+ return false, err
+ }
+ transaction, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
+ if err != nil {
+ return false, fmt.Errorf("begin fact transaction: %w", err)
+ }
+ defer func() { _ = transaction.Rollback(ctx) }()
+ if _, err := transaction.Exec(ctx, `select pg_advisory_xact_lock(hashtextextended($1, 0))`, fact.LogicalKey); err != nil {
+ return false, fmt.Errorf("lock reservation fact: %w", err)
+ }
+
+ var previousID uuid.UUID
+ var previousRevision int
+ var previousSHA string
+ err = transaction.QueryRow(ctx, `select fact_id, revision, source_sha256
+ from billing_meter.reservation_hour_current where logical_key = $1`, fact.LogicalKey).
+ Scan(&previousID, &previousRevision, &previousSHA)
+ if err == nil && previousSHA == fact.SourceSHA256 {
+ if err := transaction.Commit(ctx); err != nil {
+ return false, fmt.Errorf("commit unchanged reservation fact: %w", err)
+ }
+ return false, nil
+ }
+ if err != nil && !errors.Is(err, pgx.ErrNoRows) {
+ return false, fmt.Errorf("read current reservation fact: %w", err)
+ }
+
+ revision := 1
+ var supersedes any
+ if err == nil {
+ revision = previousRevision + 1
+ supersedes = previousID
+ }
+ _, err = transaction.Exec(ctx, `insert into billing_meter.reservation_hour_fact (
+ fact_id, logical_key, revision, cluster_id, capsule_tenant, namespace,
+ sandbox_uid, sandbox_name, pool_name, runtime, hour_start, hour_end,
+ virtual_cpu_core_seconds, virtual_memory_byte_seconds, ready_seconds,
+ covered_seconds, scrape_interval_seconds, source_sha256, collection_run_id,
+ supersedes_fact_id
+ ) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16,$17,$18,$19,$20)`,
+ fact.FactID, fact.LogicalKey, revision, fact.ClusterID, fact.CapsuleTenant, fact.Namespace,
+ fact.SandboxUID, fact.SandboxName, fact.PoolName, fact.Runtime, fact.HourStart, fact.HourEnd,
+ fact.VirtualCPUCoreSeconds, fact.VirtualMemoryByteSeconds, fact.ReadySeconds,
+ fact.CoveredSeconds, fact.ScrapeIntervalSeconds, fact.SourceSHA256, fact.CollectionRunID,
+ supersedes)
+ if err != nil {
+ return false, fmt.Errorf("insert reservation fact: %w", err)
+ }
+ if err := transaction.Commit(ctx); err != nil {
+ return false, fmt.Errorf("commit reservation fact: %w", err)
+ }
+ return true, nil
+}
+
+func (s *PostgresStore) CompleteHour(ctx context.Context, completion HourCompletion) (bool, error) {
+ if err := validateHourCompletion(completion); err != nil {
+ return false, err
+ }
+ transaction, err := s.pool.BeginTx(ctx, pgx.TxOptions{})
+ if err != nil {
+ return false, fmt.Errorf("begin hour completion transaction: %w", err)
+ }
+ defer func() { _ = transaction.Rollback(ctx) }()
+ if _, err := transaction.Exec(ctx, `select pg_advisory_xact_lock(hashtextextended(, 0))`, completion.LogicalKey); err != nil {
+ return false, fmt.Errorf("lock reservation hour completion: %w", err)
+ }
+ var previousID uuid.UUID
+ var previousRevision int
+ var previousSHA string
+ err = transaction.QueryRow(ctx, `select collection_run_id, revision, source_sha256
+ from billing_meter.reservation_hour_collection_current where logical_key = `, completion.LogicalKey).
+ Scan(&previousID, &previousRevision, &previousSHA)
+ if err == nil && previousSHA == completion.SourceSHA256 {
+ if err := transaction.Commit(ctx); err != nil {
+ return false, fmt.Errorf("commit unchanged hour completion: %w", err)
+ }
+ return false, nil
+ }
+ if err != nil && !errors.Is(err, pgx.ErrNoRows) {
+ return false, fmt.Errorf("read current hour completion: %w", err)
+ }
+ revision := 1
+ var supersedes any
+ if err == nil {
+ revision = previousRevision + 1
+ supersedes = previousID
+ }
+ _, err = transaction.Exec(ctx, `insert into billing_meter.reservation_hour_collection (
+ collection_run_id, logical_key, revision, cluster_id, hour_start, hour_end,
+ covered_seconds, discovered_sandboxes, inserted_facts, unchanged_facts,
+ source_sha256, supersedes_collection_run_id
+ ) values (,,,,,,,,,,,)`,
+ completion.CollectionRunID, completion.LogicalKey, revision, completion.ClusterID,
+ completion.HourStart, completion.HourEnd, completion.CoveredSeconds,
+ completion.DiscoveredSandboxes, completion.InsertedFacts, completion.UnchangedFacts,
+ completion.SourceSHA256, supersedes)
+ if err != nil {
+ return false, fmt.Errorf("insert hour completion: %w", err)
+ }
+ if err := transaction.Commit(ctx); err != nil {
+ return false, fmt.Errorf("commit hour completion: %w", err)
+ }
+ return true, nil
+}
+
+func validateHourCompletion(completion HourCompletion) error {
+ if completion.CollectionRunID == uuid.Nil || completion.LogicalKey == "" || completion.ClusterID == "" || len(completion.SourceSHA256) != 64 {
+ return fmt.Errorf("hour completion has empty required fields")
+ }
+ if !completion.HourEnd.Equal(completion.HourStart.Add(time.Hour)) || !completion.HourStart.Equal(completion.HourStart.UTC().Truncate(time.Hour)) {
+ return fmt.Errorf("hour completion has invalid hour")
+ }
+ if completion.CoveredSeconds < 0 || completion.CoveredSeconds > 3600 || completion.DiscoveredSandboxes < 0 || completion.InsertedFacts < 0 || completion.UnchangedFacts < 0 {
+ return fmt.Errorf("hour completion has invalid quantities")
+ }
+ return nil
+}
+
+func validateHourFact(fact HourFact) error {
+ if fact.FactID == uuid.Nil || fact.CollectionRunID == uuid.Nil || fact.LogicalKey == "" || fact.ClusterID == "" || fact.CapsuleTenant == "" || fact.Namespace == "" || fact.SandboxUID == "" || fact.SandboxName == "" || fact.PoolName == "" || fact.Runtime == "" {
+ return fmt.Errorf("reservation fact has empty required fields")
+ }
+ if !fact.HourEnd.Equal(fact.HourStart.Add(time.Hour)) || !fact.HourStart.Equal(fact.HourStart.UTC().Truncate(time.Hour)) {
+ return fmt.Errorf("reservation fact has invalid hour")
+ }
+ if len(fact.SourceSHA256) != 64 || fact.ScrapeIntervalSeconds <= 0 || fact.VirtualCPUCoreSeconds < 0 || fact.VirtualMemoryByteSeconds < 0 || fact.ReadySeconds < 0 || fact.CoveredSeconds < 0 || fact.CoveredSeconds > 3600 {
+ return fmt.Errorf("reservation fact has invalid quantities")
+ }
+ return nil
+}
+
+var _ TenantResolver = (*PostgresStore)(nil)
+var _ FactWriter = (*PostgresStore)(nil)
diff --git a/libs/fleet/backend/metering/prometheus.go b/libs/fleet/backend/metering/prometheus.go
new file mode 100644
index 0000000000..7dcdea5327
--- /dev/null
+++ b/libs/fleet/backend/metering/prometheus.go
@@ -0,0 +1,126 @@
+package metering
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+)
+
+const maxPrometheusResponseBytes = 32 << 20
+
+type Series struct {
+ Labels map[string]string
+ Samples []Sample
+}
+
+type PrometheusClient struct {
+ baseURL *url.URL
+ client *http.Client
+}
+
+func NewPrometheusClient(rawURL string, client *http.Client) (*PrometheusClient, error) {
+ baseURL, err := url.Parse(rawURL)
+ if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" {
+ return nil, fmt.Errorf("invalid Prometheus URL")
+ }
+ if client == nil {
+ client = &http.Client{Timeout: 30 * time.Second}
+ }
+ return &PrometheusClient{baseURL: baseURL, client: client}, nil
+}
+
+func (c *PrometheusClient) QueryRange(ctx context.Context, query string, start, end time.Time, step time.Duration) ([]Series, error) {
+ if strings.TrimSpace(query) == "" || !end.After(start) || step <= 0 {
+ return nil, fmt.Errorf("invalid Prometheus range query")
+ }
+ endpoint := *c.baseURL
+ endpoint.Path = strings.TrimRight(endpoint.Path, "/") + "/api/v1/query_range"
+ values := endpoint.Query()
+ values.Set("query", query)
+ values.Set("start", formatPrometheusTime(start))
+ values.Set("end", formatPrometheusTime(end))
+ values.Set("step", strconv.FormatFloat(step.Seconds(), 'f', -1, 64))
+ endpoint.RawQuery = values.Encode()
+
+ request, err := http.NewRequestWithContext(ctx, http.MethodGet, endpoint.String(), nil)
+ if err != nil {
+ return nil, fmt.Errorf("build Prometheus request: %w", err)
+ }
+ response, err := c.client.Do(request)
+ if err != nil {
+ return nil, fmt.Errorf("query Prometheus: %w", err)
+ }
+ defer response.Body.Close()
+ body, err := io.ReadAll(io.LimitReader(response.Body, maxPrometheusResponseBytes+1))
+ if err != nil {
+ return nil, fmt.Errorf("read Prometheus response: %w", err)
+ }
+ if len(body) > maxPrometheusResponseBytes {
+ return nil, fmt.Errorf("Prometheus response exceeds %d bytes", maxPrometheusResponseBytes)
+ }
+ if response.StatusCode != http.StatusOK {
+ return nil, fmt.Errorf("Prometheus returned HTTP %d", response.StatusCode)
+ }
+
+ var payload struct {
+ Status string `json:"status"`
+ Data struct {
+ ResultType string `json:"resultType"`
+ Result []struct {
+ Metric map[string]string `json:"metric"`
+ Values [][]json.RawMessage `json:"values"`
+ } `json:"result"`
+ } `json:"data"`
+ Error string `json:"error"`
+ }
+ if err := json.Unmarshal(body, &payload); err != nil {
+ return nil, fmt.Errorf("decode Prometheus response: %w", err)
+ }
+ if payload.Status != "success" || payload.Data.ResultType != "matrix" {
+ return nil, fmt.Errorf("Prometheus range query failed")
+ }
+
+ series := make([]Series, 0, len(payload.Data.Result))
+ for _, result := range payload.Data.Result {
+ parsed := Series{Labels: result.Metric, Samples: make([]Sample, 0, len(result.Values))}
+ for _, pair := range result.Values {
+ if len(pair) != 2 {
+ return nil, fmt.Errorf("invalid Prometheus sample pair")
+ }
+ var timestamp float64
+ var valueText string
+ if err := json.Unmarshal(pair[0], &timestamp); err != nil {
+ return nil, fmt.Errorf("decode Prometheus timestamp: %w", err)
+ }
+ if err := json.Unmarshal(pair[1], &valueText); err != nil {
+ return nil, fmt.Errorf("decode Prometheus value: %w", err)
+ }
+ value, err := strconv.ParseFloat(valueText, 64)
+ if err != nil {
+ return nil, fmt.Errorf("parse Prometheus value: %w", err)
+ }
+ seconds, fraction := mathModf(timestamp)
+ parsed.Samples = append(parsed.Samples, Sample{
+ Timestamp: time.Unix(seconds, int64(fraction*float64(time.Second))).UTC(),
+ Value: value,
+ })
+ }
+ series = append(series, parsed)
+ }
+ return series, nil
+}
+
+func formatPrometheusTime(value time.Time) string {
+ return strconv.FormatFloat(float64(value.UnixNano())/float64(time.Second), 'f', 3, 64)
+}
+
+func mathModf(value float64) (int64, float64) {
+ seconds := int64(value)
+ return seconds, value - float64(seconds)
+}
diff --git a/libs/fleet/backend/metering/prometheus_test.go b/libs/fleet/backend/metering/prometheus_test.go
new file mode 100644
index 0000000000..9127de9056
--- /dev/null
+++ b/libs/fleet/backend/metering/prometheus_test.go
@@ -0,0 +1,35 @@
+package metering
+
+import (
+ "context"
+ "net/http"
+ "net/http/httptest"
+ "testing"
+ "time"
+)
+
+func TestPrometheusClientParsesMatrix(t *testing.T) {
+ server := httptest.NewServer(http.HandlerFunc(func(response http.ResponseWriter, request *http.Request) {
+ if request.URL.Path != "/api/v1/query_range" || request.URL.Query().Get("query") != "metric_name" {
+ t.Fatalf("unexpected request: %s", request.URL.String())
+ }
+ response.Header().Set("Content-Type", "application/json")
+ _, _ = response.Write([]byte(`{"status":"success","data":{"resultType":"matrix","result":[{"metric":{"sandbox_uid":"uid-1"},"values":[[1787382000.5,"4"]]}]}}`))
+ }))
+ defer server.Close()
+
+ client, err := NewPrometheusClient(server.URL, server.Client())
+ if err != nil {
+ t.Fatal(err)
+ }
+ series, err := client.QueryRange(context.Background(), "metric_name", time.Unix(1787382000, 0), time.Unix(1787382060, 0), 15*time.Second)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(series) != 1 || series[0].Labels["sandbox_uid"] != "uid-1" || len(series[0].Samples) != 1 || series[0].Samples[0].Value != 4 {
+ t.Fatalf("series = %#v", series)
+ }
+ if got := series[0].Samples[0].Timestamp.Nanosecond(); got != 500000000 {
+ t.Fatalf("timestamp nanos = %d, want 500000000", got)
+ }
+}
diff --git a/libs/fleet/backend/usage/postgres.go b/libs/fleet/backend/usage/postgres.go
index 77b2e5d882..0b6e389ba8 100644
--- a/libs/fleet/backend/usage/postgres.go
+++ b/libs/fleet/backend/usage/postgres.go
@@ -94,3 +94,38 @@ func validateSandboxEvent(event SandboxEvent) error {
}
var _ EventStore = (*PostgresEventStore)(nil)
+
+const reservationFactsQuery = `select namespace, sandbox_uid, sandbox_name, pool_name, runtime,
+ hour_start, hour_end, virtual_cpu_core_seconds, virtual_memory_byte_seconds,
+ ready_seconds, covered_seconds
+from k8s_reporting.reservation_hour_facts($1, $2, $3)`
+
+func (s *PostgresEventStore) Reservations(ctx context.Context, tenant, cluster string, start, end time.Time) ([]ReservationFact, time.Time, bool, error) {
+ if strings.TrimSpace(tenant) == "" || strings.TrimSpace(cluster) == "" || start.IsZero() || end.IsZero() || !end.After(start) {
+ return nil, time.Time{}, false, fmt.Errorf("invalid reservation query")
+ }
+ var asOf time.Time
+ var complete bool
+ if err := s.pool.QueryRow(ctx, `select data_as_of, complete from k8s_reporting.reservation_meter_status(, , )`, cluster, start, end).Scan(&asOf, &complete); err != nil {
+ return nil, time.Time{}, false, fmt.Errorf("query reservation meter status: %w", err)
+ }
+ rows, err := s.pool.Query(ctx, reservationFactsQuery, tenant, start, end)
+ if err != nil {
+ return nil, time.Time{}, false, fmt.Errorf("query reservation facts: %w", err)
+ }
+ defer rows.Close()
+ facts := make([]ReservationFact, 0)
+ for rows.Next() {
+ var fact ReservationFact
+ if err := rows.Scan(&fact.Namespace, &fact.SandboxUID, &fact.SandboxName, &fact.PoolName, &fact.Runtime, &fact.HourStart, &fact.HourEnd, &fact.VirtualCPUCoreSeconds, &fact.VirtualMemoryByteSeconds, &fact.ReadySeconds, &fact.CoveredSeconds); err != nil {
+ return nil, time.Time{}, false, fmt.Errorf("scan reservation fact: %w", err)
+ }
+ facts = append(facts, fact)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, time.Time{}, false, fmt.Errorf("read reservation facts: %w", err)
+ }
+ return facts, asOf.UTC(), !complete, nil
+}
+
+var _ ReservationStore = (*PostgresEventStore)(nil)
diff --git a/libs/fleet/backend/usage/provider.go b/libs/fleet/backend/usage/provider.go
index 5a35f25e20..eb3fca84ac 100644
--- a/libs/fleet/backend/usage/provider.go
+++ b/libs/fleet/backend/usage/provider.go
@@ -27,13 +27,15 @@ const (
)
type provider struct {
- events EventStore
- allocations AllocationClient
- clock func() time.Time
- cacheTTL time.Duration
- cacheMu sync.Mutex
- cache map[usageCacheKey]usageCacheEntry
- loads singleflight.Group
+ events EventStore
+ allocations AllocationClient
+ reservations ReservationStore
+ reservationCluster string
+ clock func() time.Time
+ cacheTTL time.Duration
+ cacheMu sync.Mutex
+ cache map[usageCacheKey]usageCacheEntry
+ loads singleflight.Group
}
type usageCacheKey struct {
@@ -58,11 +60,15 @@ type liveSegment struct {
}
func NewProvider(events EventStore, allocations AllocationClient, clock func() time.Time) *provider {
+ return NewProviderWithReservations(events, allocations, nil, "", clock)
+}
+
+func NewProviderWithReservations(events EventStore, allocations AllocationClient, reservations ReservationStore, reservationCluster string, clock func() time.Time) *provider {
if clock == nil {
clock = time.Now
}
return &provider{
- events: events, allocations: allocations, clock: clock,
+ events: events, allocations: allocations, reservations: reservations, reservationCluster: reservationCluster, clock: clock,
cacheTTL: defaultUsageCacheTTL, cache: make(map[usageCacheKey]usageCacheEntry),
}
}
@@ -269,6 +275,26 @@ func (p *provider) loadUncached(ctx context.Context, query Query, now time.Time)
data.partial = true
}
}
+ if p.reservations != nil {
+ reservationCtx, reservationSpan := usageTracer().Start(ctx, "usage.reservations.query")
+ facts, reservationAsOf, reservationPartial, reservationErr := p.reservations.Reservations(reservationCtx, identity.PersonalGroup(reservationCtx, query.Subject), p.reservationCluster, start, end)
+ if reservationErr != nil {
+ markUsageSpanError(reservationSpan, "reservation fact query failed")
+ reservationSpan.End()
+ markUsageSpanError(span, "reservation fact query failed")
+ return usageData{}, fmt.Errorf("read reservation facts: %w", reservationErr)
+ }
+ reservationSpan.SetAttributes(attribute.Int("usage.reservation_fact_count", len(facts)))
+ reservationSpan.End()
+ if reservationAsOf.Before(data.asOf) {
+ data.asOf = maxTime(start, reservationAsOf.UTC())
+ }
+ data.partial = data.partial || reservationPartial || data.asOf.Before(end)
+ if err := applyReservationFacts(facts, start, end, step, poolTotals, data.buckets); err != nil {
+ markUsageSpanError(span, "reservation fact attribution failed")
+ return usageData{}, err
+ }
+ }
for _, totals := range poolTotals {
if !totals.isZero() {
data.pools = append(data.pools, totals)
@@ -641,3 +667,35 @@ func maxTime(left, right time.Time) time.Time {
}
var _ Provider = (*provider)(nil)
+
+func applyReservationFacts(facts []ReservationFact, windowStart, windowEnd time.Time, bucketDuration time.Duration, pools map[string]usageTotals, buckets map[bucketKey]usageTotals) error {
+ for id, pool := range pools {
+ pool.cpuProvisioned = 0
+ pool.memoryProvisioned = 0
+ pools[id] = pool
+ }
+ for key, bucket := range buckets {
+ bucket.cpuProvisioned = 0
+ bucket.memoryProvisioned = 0
+ buckets[key] = bucket
+ }
+ for _, fact := range facts {
+ if fact.Namespace == "" || !validPoolName(fact.PoolName) || !fact.HourEnd.Equal(fact.HourStart.Add(time.Hour)) || fact.HourStart.Before(windowStart) || fact.HourEnd.After(windowEnd) || fact.VirtualCPUCoreSeconds < 0 || fact.VirtualMemoryByteSeconds < 0 {
+ return fmt.Errorf("invalid reservation fact")
+ }
+ cpuCoreHours := fact.VirtualCPUCoreSeconds / 3600
+ memoryGiBHours := fact.VirtualMemoryByteSeconds / 3600 / gibibyte
+ id := fact.Namespace + ":" + fact.PoolName
+ pool := pools[id]
+ pool.id, pool.name = id, fact.PoolName
+ pool.cpuProvisioned += cpuCoreHours
+ pool.memoryProvisioned += memoryGiBHours
+ pools[id] = pool
+ key := bucketKey{namespace: fact.Namespace, pool: fact.PoolName, start: bucketStartFor(fact.HourStart, windowStart, bucketDuration)}
+ bucket := buckets[key]
+ bucket.cpuProvisioned += cpuCoreHours
+ bucket.memoryProvisioned += memoryGiBHours
+ buckets[key] = bucket
+ }
+ return nil
+}
diff --git a/libs/fleet/backend/usage/provider_test.go b/libs/fleet/backend/usage/provider_test.go
index a886ded87c..a67722f483 100644
--- a/libs/fleet/backend/usage/provider_test.go
+++ b/libs/fleet/backend/usage/provider_test.go
@@ -388,3 +388,49 @@ func TestProviderRestoresAttributionAfterUnambiguousAdded(t *testing.T) {
t.Fatalf("overview=%#v err=%v", got, err)
}
}
+
+type fakeReservationStore struct {
+ facts []ReservationFact
+ asOf time.Time
+ partial bool
+ err error
+ tenant string
+ cluster string
+}
+
+func (s *fakeReservationStore) Reservations(_ context.Context, tenant, cluster string, _, _ time.Time) ([]ReservationFact, time.Time, bool, error) {
+ s.tenant, s.cluster = tenant, cluster
+ return s.facts, s.asOf, s.partial, s.err
+}
+
+func TestProviderUsesVirtualReservationsForProvisionedTotals(t *testing.T) {
+ t.Parallel()
+ cutoff := time.Date(2026, 8, 22, 8, 0, 0, 0, time.UTC)
+ events := &fakeEventStore{events: []SandboxEvent{{
+ EventID: mustUUID("00000000-0000-0000-0000-000000000090"), Namespace: "ns-a", SandboxName: "sandbox-a", SandboxUID: "uid-a",
+ PoolName: "pool-a", Runtime: "kubevirt", VMName: "vm-a", EventType: "Added", ObservedAt: cutoff.Add(-25 * time.Hour),
+ }}}
+ allocations := &fakeAllocationClient{asOf: cutoff, allocations: []Allocation{{
+ Start: cutoff.Add(-time.Hour), End: cutoff, Namespace: "ns-a", Pod: "virt-launcher-vm-a-abc",
+ Minutes: 60, CPUUsageAverage: 1, CPURequestAverage: 2,
+ RAMUsageAverageBytes: 3 * gibibyte, RAMRequestAverageBytes: 4 * gibibyte, CostUSD: 0.5,
+ }}}
+ reservations := &fakeReservationStore{asOf: cutoff, facts: []ReservationFact{{
+ Namespace: "ns-a", SandboxUID: "uid-a", SandboxName: "sandbox-a", PoolName: "pool-a", Runtime: "kubevirt",
+ HourStart: cutoff.Add(-time.Hour), HourEnd: cutoff,
+ VirtualCPUCoreSeconds: 4 * 3600, VirtualMemoryByteSeconds: 8 * gibibyte * 3600,
+ }}}
+ provider := NewProviderWithReservations(events, allocations, reservations, "kopf-k3s", func() time.Time { return cutoff })
+
+ got, err := provider.Overview(context.Background(), Query{Subject: "alice", Timeframe: Timeframe24H})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if reservations.tenant != "user-alice" || reservations.cluster != "kopf-k3s" || got.Partial || len(got.Pools) != 1 {
+ t.Fatalf("overview=%#v reservations=%+v", got, reservations)
+ }
+ pool := got.Pools[0]
+ if pool.CPU.Consumed != 1 || pool.CPU.Provisioned != 4 || pool.Memory.Consumed != 3 || pool.Memory.Provisioned != 8 || pool.CostUSD != 0.5 {
+ t.Fatalf("pool=%#v", pool)
+ }
+}
diff --git a/libs/fleet/backend/usage/types.go b/libs/fleet/backend/usage/types.go
index d8bd0464fe..7fce917ef7 100644
--- a/libs/fleet/backend/usage/types.go
+++ b/libs/fleet/backend/usage/types.go
@@ -112,3 +112,21 @@ type AllocationClient interface {
}
const gibibyte = float64(1 << 30)
+
+type ReservationFact struct {
+ Namespace string
+ SandboxUID string
+ SandboxName string
+ PoolName string
+ Runtime string
+ HourStart time.Time
+ HourEnd time.Time
+ VirtualCPUCoreSeconds float64
+ VirtualMemoryByteSeconds float64
+ ReadySeconds float64
+ CoveredSeconds float64
+}
+
+type ReservationStore interface {
+ Reservations(context.Context, string, string, time.Time, time.Time) ([]ReservationFact, time.Time, bool, error)
+}