Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

feat(billing): Stripe usage submission ledger migration and submitter role (CUA-1158) (#7873)

ddupont8cb8f6d2b927
diff --git a/libs/fleet/backend/cmd/db-migrate/main.go b/libs/fleet/backend/cmd/db-migrate/main.go
index bb061c3593..48a87c234a 100644
--- a/libs/fleet/backend/cmd/db-migrate/main.go
+++ b/libs/fleet/backend/cmd/db-migrate/main.go
@@ -37,6 +37,7 @@ func main() {
Metabase: required("METABASE_DATABASE_URL"),
Usage: required("USAGE_DATABASE_URL"),
Meter: required("METER_DATABASE_URL"),
+ Submitter: required("SUBMITTER_DATABASE_URL"),
},
}
if err := database.Run(context.Background(), config); err != nil {
diff --git a/libs/fleet/backend/database/migrations/000013_stripe_usage_submission.sql b/libs/fleet/backend/database/migrations/000013_stripe_usage_submission.sql
new file mode 100644
index 0000000000..fa5ce4bffb
--- /dev/null
+++ b/libs/fleet/backend/database/migrations/000013_stripe_usage_submission.sql
@@ -0,0 +1,116 @@
+-- Stripe usage submission ledger (CUA-1158).
+--
+-- Records every meter event the usage submitter archives and sends so that
+-- replays, revisions, and out-of-window corrections are provable locally.
+-- Stripe's identifier dedupe is only guaranteed for 24 hours; this ledger is
+-- the durable exactly-once record. Design:
+-- docs/superpowers/specs/2026-09-16-cua-1158-stripe-usage-submitter-design.md
+--
+-- Touches no existing table. Creates two empty tables and one login role.
+
+CREATE ROLE cyclops_submitter LOGIN NOINHERIT NOCREATEROLE NOSUPERUSER NOCREATEDB NOREPLICATION NOBYPASSRLS;
+
+SET LOCAL ROLE billing_meter_owner;
+
+CREATE TABLE billing_meter.stripe_submission_batch (
+ batch_id uuid PRIMARY KEY,
+ cluster_id text NOT NULL CHECK (cluster_id <> ''),
+ cutover_hour timestamptz NOT NULL CHECK (date_trunc('hour', cutover_hour) = cutover_hour),
+ window_start timestamptz NOT NULL CHECK (date_trunc('hour', window_start) = window_start),
+ window_end timestamptz NOT NULL CHECK (date_trunc('hour', window_end) = window_end),
+ dry_run boolean NOT NULL,
+ archive_bucket text NOT NULL CHECK (archive_bucket <> ''),
+ archive_key text NOT NULL UNIQUE CHECK (archive_key <> ''),
+ archive_sha256 text CHECK (archive_sha256 IS NULL OR archive_sha256 ~ '^[0-9a-f]{64}$'),
+ submission_count integer NOT NULL CHECK (submission_count >= 0),
+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+ archived_at timestamptz,
+ completed_at timestamptz,
+ CHECK (window_end > window_start),
+ CHECK (window_start >= cutover_hour),
+ CHECK ((archived_at IS NULL) = (archive_sha256 IS NULL)),
+ CHECK (completed_at IS NULL OR archived_at IS NOT NULL)
+);
+
+CREATE TABLE billing_meter.stripe_submission (
+ submission_id uuid PRIMARY KEY,
+ batch_id uuid NOT NULL REFERENCES billing_meter.stripe_submission_batch(batch_id),
+ capsule_tenant text NOT NULL CHECK (capsule_tenant <> ''),
+ hour_start timestamptz NOT NULL CHECK (date_trunc('hour', hour_start) = hour_start),
+ meter text NOT NULL CHECK (meter IN ('cua_vcpu_hours', 'cua_gib_hours')),
+ seq integer NOT NULL CHECK (seq > 0),
+ identifier text NOT NULL UNIQUE CHECK (identifier <> '' AND length(identifier) <= 100),
+ stripe_customer_id text CHECK (stripe_customer_id IS NULL OR stripe_customer_id <> ''),
+ quantity numeric(20, 6) NOT NULL CHECK (quantity >= 0),
+ fact_set_sha256 text NOT NULL CHECK (fact_set_sha256 ~ '^[0-9a-f]{64}$'),
+ status text NOT NULL CHECK (status IN ('archived', 'sent', 'dry_run', 'failed', 'out_of_window', 'unmapped')),
+ stripe_event_id text CHECK (stripe_event_id IS NULL OR stripe_event_id <> ''),
+ error_code text,
+ error_message text,
+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),
+ sent_at timestamptz,
+ UNIQUE (capsule_tenant, hour_start, meter, seq),
+ CHECK (status <> 'sent' OR (stripe_event_id IS NOT NULL AND sent_at IS NOT NULL AND stripe_customer_id IS NOT NULL)),
+ CHECK (status <> 'unmapped' OR stripe_customer_id IS NULL),
+ CHECK (status NOT IN ('failed', 'out_of_window') OR error_code IS NOT NULL)
+);
+
+CREATE INDEX stripe_submission_tenant_hour_meter_idx
+ ON billing_meter.stripe_submission (capsule_tenant, hour_start, meter, seq DESC);
+CREATE INDEX stripe_submission_status_idx
+ ON billing_meter.stripe_submission (status, hour_start);
+
+-- Status moves forward only. 'archived' is the only non-terminal state.
+CREATE FUNCTION billing_meter.enforce_stripe_submission_transition()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+BEGIN
+ IF OLD.status <> 'archived' THEN
+ RAISE EXCEPTION 'stripe submission % is terminal in status %', OLD.identifier, OLD.status;
+ END IF;
+ IF NEW.status = 'archived' THEN
+ RAISE EXCEPTION 'stripe submission % cannot return to archived', OLD.identifier;
+ END IF;
+ IF NEW.submission_id <> OLD.submission_id OR NEW.batch_id <> OLD.batch_id
+ OR NEW.capsule_tenant <> OLD.capsule_tenant OR NEW.hour_start <> OLD.hour_start
+ OR NEW.meter <> OLD.meter OR NEW.seq <> OLD.seq OR NEW.identifier <> OLD.identifier
+ OR NEW.quantity <> OLD.quantity OR NEW.fact_set_sha256 <> OLD.fact_set_sha256
+ OR NEW.created_at <> OLD.created_at THEN
+ RAISE EXCEPTION 'stripe submission % identity and quantity are immutable', OLD.identifier;
+ END IF;
+ RETURN NEW;
+END
+$$;
+
+CREATE FUNCTION billing_meter.reject_stripe_submission_removal()
+RETURNS trigger
+LANGUAGE plpgsql
+AS $$
+BEGIN
+ RAISE EXCEPTION 'billing_meter stripe submissions are append-only';
+END
+$$;
+
+CREATE TRIGGER stripe_submission_transition
+BEFORE UPDATE ON billing_meter.stripe_submission
+FOR EACH ROW EXECUTE FUNCTION billing_meter.enforce_stripe_submission_transition();
+CREATE TRIGGER stripe_submission_no_removal
+BEFORE DELETE OR TRUNCATE ON billing_meter.stripe_submission
+FOR EACH STATEMENT EXECUTE FUNCTION billing_meter.reject_stripe_submission_removal();
+CREATE TRIGGER stripe_submission_batch_no_removal
+BEFORE DELETE OR TRUNCATE ON billing_meter.stripe_submission_batch
+FOR EACH STATEMENT EXECUTE FUNCTION billing_meter.reject_stripe_submission_removal();
+
+-- Migration 000010's default privileges grant k8s_metabase SELECT on these
+-- tables, matching the repo invariant that Metabase reads every billing_meter
+-- relation (reconciliation dashboards). The migrator's expected reporting ACL
+-- list includes them.
+REVOKE ALL ON TABLE billing_meter.stripe_submission, billing_meter.stripe_submission_batch FROM PUBLIC;
+REVOKE ALL ON FUNCTION billing_meter.enforce_stripe_submission_transition() FROM PUBLIC;
+REVOKE ALL ON FUNCTION billing_meter.reject_stripe_submission_removal() FROM PUBLIC;
+
+GRANT USAGE ON SCHEMA billing_meter TO cyclops_submitter;
+GRANT SELECT ON TABLE billing_meter.reservation_hour_current, billing_meter.reservation_hour_collection_current TO cyclops_submitter;
+GRANT SELECT, INSERT, UPDATE ON TABLE billing_meter.stripe_submission, billing_meter.stripe_submission_batch TO cyclops_submitter;
+RESET ROLE;
diff --git a/libs/fleet/backend/database/migrator.go b/libs/fleet/backend/database/migrator.go
index 9ec854e3b8..57dbfa006a 100644
--- a/libs/fleet/backend/database/migrator.go
+++ b/libs/fleet/backend/database/migrator.go
@@ -55,6 +55,7 @@ type CredentialURLs struct {
Metabase string
Usage string
Meter string
+ Submitter string
}
type migrationFile struct {
@@ -163,6 +164,7 @@ var expectedCredentialRoles = map[string]string{
"metabase": "k8s_metabase",
"usage": "cyclops_usage_reader",
"meter": "cyclops_meter_writer",
+ "submitter": "cyclops_submitter",
}
const selectAppliedMigrationsStatement = `select version, filename, sha256 from cyclops_migrations.applied_migrations order by application_order`
@@ -484,6 +486,7 @@ func parseCredentialURLs(urls CredentialURLs) ([]credential, error) {
{Name: "metabase", URL: urls.Metabase},
{Name: "usage", URL: urls.Usage},
{Name: "meter", URL: urls.Meter},
+ {Name: "submitter", URL: urls.Submitter},
}
credentials := make([]credential, 0, len(inputs))
@@ -519,6 +522,7 @@ func staticRoleContracts() []staticRoleContract {
{role: "k8s_metabase", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
{role: "cyclops_usage_reader", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
{role: "cyclops_meter_writer", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
+ {role: "cyclops_submitter", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},
}
}
@@ -795,6 +799,8 @@ func expectedReportingACLs() []reportingACL {
{object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_fact"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},
{object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_current"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},
{object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_collection_current"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},
+ {object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "stripe_submission_batch"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},
+ {object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "stripe_submission"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},
{object: reportingObject{kind: reportingObjectSchema, schema: "k8s_reporting"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeUsage, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},
{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_reporting", name: "current_resources"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},
{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_reporting", name: "hourly_reservation_usage"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},
diff --git a/libs/fleet/backend/database/migrator_pg_test.go b/libs/fleet/backend/database/migrator_pg_test.go
index 391220d0c1..fff1c0de3f 100644
--- a/libs/fleet/backend/database/migrator_pg_test.go
+++ b/libs/fleet/backend/database/migrator_pg_test.go
@@ -28,6 +28,7 @@ var staticMigrationRoles = []string{
"cyclops_app",
"cyclops_usage_reader",
"cyclops_meter_writer",
+ "cyclops_submitter",
"k8s_state_owner",
"k8s_state_writer",
"k8s_state_exporter",
@@ -91,8 +92,8 @@ func TestInitialMigrationBuildsCompleteDatabase(t *testing.T) {
firstSummary := captureRunSummary(t, func() error {
return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})
})
- if firstSummary.Pending != 12 || firstSummary.Applied != 12 {
- t.Fatalf("initial migration summary = %+v, want pending=12 applied=12", firstSummary)
+ if firstSummary.Pending != 13 || firstSummary.Applied != 13 {
+ t.Fatalf("initial migration summary = %+v, want pending=13 applied=13", firstSummary)
}
before := migrationLedgerRows(t, ctx, migrationURL)
secondSummary := captureRunSummary(t, func() error {
@@ -201,14 +202,14 @@ func TestRunUpgradesVersionOneAndThenNoOps(t *testing.T) {
upgrade := captureRunSummary(t, func() error {
return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})
})
- if upgrade.Current != 1 || upgrade.Target != 12 || upgrade.Pending != 11 || upgrade.Applied != 11 || upgrade.Skipped != 1 || upgrade.Result != "success" {
+ if upgrade.Current != 1 || upgrade.Target != 13 || upgrade.Pending != 12 || upgrade.Applied != 12 || upgrade.Skipped != 1 || upgrade.Result != "success" {
t.Fatalf("version-one upgrade summary = %+v", upgrade)
}
noOp := captureRunSummary(t, func() error {
return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})
})
- if noOp.Current != 12 || noOp.Target != 12 || noOp.Pending != 0 || noOp.Applied != 0 || noOp.Skipped != 12 || noOp.Result != "success" {
+ if noOp.Current != 13 || noOp.Target != 13 || noOp.Pending != 0 || noOp.Applied != 0 || noOp.Skipped != 13 || noOp.Result != "success" {
t.Fatalf("post-upgrade no-op summary = %+v", noOp)
}
}
@@ -1572,6 +1573,7 @@ func testCredentialURLs(t *testing.T, adminURL string) CredentialURLs {
Metabase: withRole("k8s_metabase"),
Usage: withRole("cyclops_usage_reader"),
Meter: withRole("cyclops_meter_writer"),
+ Submitter: withRole("cyclops_submitter"),
}
}
@@ -1604,8 +1606,8 @@ func migrationLedgerRows(t *testing.T, ctx context.Context, adminURL string) []l
if err := rows.Err(); err != nil {
t.Fatal("iterate migration ledger")
}
- if len(ledger) != 12 {
- t.Fatalf("migration ledger row count = %d, want 12", len(ledger))
+ if len(ledger) != 13 {
+ t.Fatalf("migration ledger row count = %d, want 13", len(ledger))
}
for index, want := range []struct {
version int64
@@ -1623,6 +1625,7 @@ func migrationLedgerRows(t *testing.T, ctx context.Context, adminURL string) []l
{10, "000010_grant_metabase_billing_meter_access.sql"},
{11, "000011_signed_service_urls.sql"},
{12, "000012_private_account_lookup.sql"},
+ {13, "000013_stripe_usage_submission.sql"},
} {
if ledger[index].Version != want.version || ledger[index].ApplicationOrder != int64(index+1) || ledger[index].Filename != want.filename {
t.Fatalf("migration ledger row %d = version:%d order:%d filename:%q", index, ledger[index].Version, ledger[index].ApplicationOrder, ledger[index].Filename)
@@ -1761,6 +1764,7 @@ func assertRoleContract(t *testing.T, ctx context.Context, adminURL string) {
"k8s_metabase": {true, false, false},
"cyclops_usage_reader": {true, false, false},
"cyclops_meter_writer": {true, false, false},
+ "cyclops_submitter": {true, false, false},
}
for role, want := range expected {
var login, inherit, createRole, super, createDB, replication, bypassRLS bool
@@ -2094,7 +2098,7 @@ func assertRuntimeLedgerAccess(t *testing.T, ctx context.Context, credentials Cr
var count int
err := connection.QueryRow(ctx, `select count(*) from cyclops_migrations.applied_migrations`).Scan(&count)
connection.Close(ctx)
- if err != nil || count != 12 {
+ if err != nil || count != 13 {
t.Errorf("%s ledger select = count:%d err:%v", role, count, err)
}
assertStatementFails(t, ctx, databaseURL, `insert into cyclops_migrations.applied_migrations (version, filename, sha256) values (99, 'invalid.sql', 'invalid')`)
@@ -2590,6 +2594,8 @@ func assertExactReportingACLContract(t *testing.T, ctx context.Context, connecti
('relation:billing_meter.reservation_hour_fact', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),
('relation:billing_meter.reservation_hour_current', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),
('relation:billing_meter.reservation_hour_collection_current', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),
+ ('relation:billing_meter.stripe_submission_batch', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),
+ ('relation:billing_meter.stripe_submission', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),
('schema:k8s_reporting', 'USAGE', 'k8s_metabase', 'k8s_reporting_owner', false),
('relation:k8s_reporting.current_resources', 'SELECT', 'k8s_metabase', 'k8s_reporting_owner', false),
('relation:k8s_reporting.hourly_reservation_usage', 'SELECT', 'k8s_metabase', 'k8s_reporting_owner', false),
diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.go
index 6775c96913..c033cb6175 100644
--- a/libs/fleet/backend/database/migrator_test.go
+++ b/libs/fleet/backend/database/migrator_test.go
@@ -521,8 +521,8 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {
if err != nil {
t.Fatal(err)
}
- if len(files) != 12 {
- t.Fatalf("expected exactly twelve migrations, got %d", len(files))
+ if len(files) != 13 {
+ t.Fatalf("expected exactly thirteen migrations, got %d", len(files))
}
manifest := make([]struct {
Version int64
@@ -550,6 +550,7 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {
{10, "000010_grant_metabase_billing_meter_access.sql"},
{11, "000011_signed_service_urls.sql"},
{12, "000012_private_account_lookup.sql"},
+ {13, "000013_stripe_usage_submission.sql"},
}) {
t.Fatalf("migration manifest = %#v", manifest)
}
@@ -927,12 +928,13 @@ func TestCredentialURLsExcludeDynamicTenantRoles(t *testing.T) {
Metabase: "postgres://k8s_metabase:pw@db/cyclops",
Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",
Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",
+ Submitter: "postgres://cyclops_submitter:pw@db/cyclops",
})
if err != nil {
t.Fatalf("parse fixed runtime credentials: %v", err)
}
- if len(credentials) != 7 {
- t.Fatalf("credential count = %d, want 7", len(credentials))
+ if len(credentials) != 8 {
+ t.Fatalf("credential count = %d, want 8", len(credentials))
}
for _, credential := range credentials {
if credential.Role == "k8s_query_broker" || strings.HasPrefix(credential.Role, "k8s_tenant_") {
@@ -960,6 +962,7 @@ func TestStaticRoleContractsAreAllowlistedAndFixed(t *testing.T) {
"k8s_metabase": {true, false, false, false, -1, staticRoleValidUntilInfinity},
"cyclops_usage_reader": {true, false, false, false, -1, staticRoleValidUntilInfinity},
"cyclops_meter_writer": {true, false, false, false, -1, staticRoleValidUntilInfinity},
+ "cyclops_submitter": {true, false, false, false, -1, staticRoleValidUntilInfinity},
}
if len(contracts) != len(want) {
t.Fatalf("static role contract count = %d, want %d", len(contracts), len(want))
@@ -1010,6 +1013,7 @@ func TestCredentialURLsRequireExpectedRoleNames(t *testing.T) {
Metabase: "postgres://k8s_metabase:pw@db/cyclops",
Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",
Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",
+ Submitter: "postgres://cyclops_submitter:pw@db/cyclops",
})
if err == nil {
t.Fatal("expected application role-name validation")
@@ -1028,6 +1032,7 @@ func TestCredentialURLsRejectEmptyPassword(t *testing.T) {
Metabase: "postgres://k8s_metabase:pw@db/cyclops",
Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",
Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",
+ Submitter: "postgres://cyclops_submitter:pw@db/cyclops",
})
if err == nil || err.Error() != "application credential database URL must include a password" {
t.Fatalf("expected empty password rejection, got %v", err)
@@ -1043,6 +1048,7 @@ func TestCredentialURLParseErrorsPreserveCauseForBoundaryClassification(t *testi
Metabase: "postgres://k8s_metabase:pw@db/cyclops",
Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",
Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",
+ Submitter: "postgres://cyclops_submitter:pw@db/cyclops",
})
if !errors.Is(err, ErrInvalidConfiguration) {
t.Fatalf("error = %v, want ErrInvalidConfiguration", err)
diff --git a/libs/fleet/backend/database/test-setup.sh b/libs/fleet/backend/database/test-setup.sh
index 10886bd802..518709f376 100755
--- a/libs/fleet/backend/database/test-setup.sh
+++ b/libs/fleet/backend/database/test-setup.sh
@@ -26,6 +26,7 @@ static_roles=(
k8s_metabase
cyclops_usage_reader
cyclops_meter_writer
+ cyclops_submitter
)
base_url="${admin_url%%\?*}"
@@ -87,6 +88,7 @@ role_admin_url="$(url_for k8s_role_admin role-admin-password)"
metabase_url="$(url_for k8s_metabase metabase-password)"
usage_url="$(url_for cyclops_usage_reader usage-reader-password)"
meter_url="$(url_for cyclops_meter_writer meter-writer-password)"
+submitter_url="$(url_for cyclops_submitter submitter-password)"
env_file="${CYCLOPS_TEST_DATABASE_ENV_FILE:-${GITHUB_ENV:-}}"
if [[ -z "$env_file" ]]; then
@@ -157,6 +159,7 @@ STATE_ROLE_ADMIN_DATABASE_URL="$role_admin_url" \
METABASE_DATABASE_URL="$metabase_url" \
USAGE_DATABASE_URL="$usage_url" \
METER_DATABASE_URL="$meter_url" \
+SUBMITTER_DATABASE_URL="$submitter_url" \
go run ./cmd/db-migrate
cat >> "$env_file" <<EOF_ENV