Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(billing): Stripe usage submission ledger migration and submitter role (CUA-1158) (#7873)
ddupont8cb8f6d2b927
diff --git a/libs/fleet/backend/cmd/db-migrate/main.go b/libs/fleet/backend/cmd/db-migrate/main.goindex bb061c3593..48a87c234a 100644--- a/libs/fleet/backend/cmd/db-migrate/main.go+++ b/libs/fleet/backend/cmd/db-migrate/main.go@@ -37,6 +37,7 @@ func main() {Metabase: required("METABASE_DATABASE_URL"),Usage: required("USAGE_DATABASE_URL"),Meter: required("METER_DATABASE_URL"),+ Submitter: required("SUBMITTER_DATABASE_URL"),},}if err := database.Run(context.Background(), config); err != nil {diff --git a/libs/fleet/backend/database/migrations/000013_stripe_usage_submission.sql b/libs/fleet/backend/database/migrations/000013_stripe_usage_submission.sqlnew file mode 100644index 0000000000..fa5ce4bffb--- /dev/null+++ b/libs/fleet/backend/database/migrations/000013_stripe_usage_submission.sql@@ -0,0 +1,116 @@+-- Stripe usage submission ledger (CUA-1158).+--+-- Records every meter event the usage submitter archives and sends so that+-- replays, revisions, and out-of-window corrections are provable locally.+-- Stripe's identifier dedupe is only guaranteed for 24 hours; this ledger is+-- the durable exactly-once record. Design:+-- docs/superpowers/specs/2026-09-16-cua-1158-stripe-usage-submitter-design.md+--+-- Touches no existing table. Creates two empty tables and one login role.++CREATE ROLE cyclops_submitter LOGIN NOINHERIT NOCREATEROLE NOSUPERUSER NOCREATEDB NOREPLICATION NOBYPASSRLS;++SET LOCAL ROLE billing_meter_owner;++CREATE TABLE billing_meter.stripe_submission_batch (+ batch_id uuid PRIMARY KEY,+ cluster_id text NOT NULL CHECK (cluster_id <> ''),+ cutover_hour timestamptz NOT NULL CHECK (date_trunc('hour', cutover_hour) = cutover_hour),+ window_start timestamptz NOT NULL CHECK (date_trunc('hour', window_start) = window_start),+ window_end timestamptz NOT NULL CHECK (date_trunc('hour', window_end) = window_end),+ dry_run boolean NOT NULL,+ archive_bucket text NOT NULL CHECK (archive_bucket <> ''),+ archive_key text NOT NULL UNIQUE CHECK (archive_key <> ''),+ archive_sha256 text CHECK (archive_sha256 IS NULL OR archive_sha256 ~ '^[0-9a-f]{64}$'),+ submission_count integer NOT NULL CHECK (submission_count >= 0),+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),+ archived_at timestamptz,+ completed_at timestamptz,+ CHECK (window_end > window_start),+ CHECK (window_start >= cutover_hour),+ CHECK ((archived_at IS NULL) = (archive_sha256 IS NULL)),+ CHECK (completed_at IS NULL OR archived_at IS NOT NULL)+);++CREATE TABLE billing_meter.stripe_submission (+ submission_id uuid PRIMARY KEY,+ batch_id uuid NOT NULL REFERENCES billing_meter.stripe_submission_batch(batch_id),+ capsule_tenant text NOT NULL CHECK (capsule_tenant <> ''),+ hour_start timestamptz NOT NULL CHECK (date_trunc('hour', hour_start) = hour_start),+ meter text NOT NULL CHECK (meter IN ('cua_vcpu_hours', 'cua_gib_hours')),+ seq integer NOT NULL CHECK (seq > 0),+ identifier text NOT NULL UNIQUE CHECK (identifier <> '' AND length(identifier) <= 100),+ stripe_customer_id text CHECK (stripe_customer_id IS NULL OR stripe_customer_id <> ''),+ quantity numeric(20, 6) NOT NULL CHECK (quantity >= 0),+ fact_set_sha256 text NOT NULL CHECK (fact_set_sha256 ~ '^[0-9a-f]{64}$'),+ status text NOT NULL CHECK (status IN ('archived', 'sent', 'dry_run', 'failed', 'out_of_window', 'unmapped')),+ stripe_event_id text CHECK (stripe_event_id IS NULL OR stripe_event_id <> ''),+ error_code text,+ error_message text,+ created_at timestamptz NOT NULL DEFAULT clock_timestamp(),+ sent_at timestamptz,+ UNIQUE (capsule_tenant, hour_start, meter, seq),+ CHECK (status <> 'sent' OR (stripe_event_id IS NOT NULL AND sent_at IS NOT NULL AND stripe_customer_id IS NOT NULL)),+ CHECK (status <> 'unmapped' OR stripe_customer_id IS NULL),+ CHECK (status NOT IN ('failed', 'out_of_window') OR error_code IS NOT NULL)+);++CREATE INDEX stripe_submission_tenant_hour_meter_idx+ ON billing_meter.stripe_submission (capsule_tenant, hour_start, meter, seq DESC);+CREATE INDEX stripe_submission_status_idx+ ON billing_meter.stripe_submission (status, hour_start);++-- Status moves forward only. 'archived' is the only non-terminal state.+CREATE FUNCTION billing_meter.enforce_stripe_submission_transition()+RETURNS trigger+LANGUAGE plpgsql+AS $$+BEGIN+ IF OLD.status <> 'archived' THEN+ RAISE EXCEPTION 'stripe submission % is terminal in status %', OLD.identifier, OLD.status;+ END IF;+ IF NEW.status = 'archived' THEN+ RAISE EXCEPTION 'stripe submission % cannot return to archived', OLD.identifier;+ END IF;+ IF NEW.submission_id <> OLD.submission_id OR NEW.batch_id <> OLD.batch_id+ OR NEW.capsule_tenant <> OLD.capsule_tenant OR NEW.hour_start <> OLD.hour_start+ OR NEW.meter <> OLD.meter OR NEW.seq <> OLD.seq OR NEW.identifier <> OLD.identifier+ OR NEW.quantity <> OLD.quantity OR NEW.fact_set_sha256 <> OLD.fact_set_sha256+ OR NEW.created_at <> OLD.created_at THEN+ RAISE EXCEPTION 'stripe submission % identity and quantity are immutable', OLD.identifier;+ END IF;+ RETURN NEW;+END+$$;++CREATE FUNCTION billing_meter.reject_stripe_submission_removal()+RETURNS trigger+LANGUAGE plpgsql+AS $$+BEGIN+ RAISE EXCEPTION 'billing_meter stripe submissions are append-only';+END+$$;++CREATE TRIGGER stripe_submission_transition+BEFORE UPDATE ON billing_meter.stripe_submission+FOR EACH ROW EXECUTE FUNCTION billing_meter.enforce_stripe_submission_transition();+CREATE TRIGGER stripe_submission_no_removal+BEFORE DELETE OR TRUNCATE ON billing_meter.stripe_submission+FOR EACH STATEMENT EXECUTE FUNCTION billing_meter.reject_stripe_submission_removal();+CREATE TRIGGER stripe_submission_batch_no_removal+BEFORE DELETE OR TRUNCATE ON billing_meter.stripe_submission_batch+FOR EACH STATEMENT EXECUTE FUNCTION billing_meter.reject_stripe_submission_removal();++-- Migration 000010's default privileges grant k8s_metabase SELECT on these+-- tables, matching the repo invariant that Metabase reads every billing_meter+-- relation (reconciliation dashboards). The migrator's expected reporting ACL+-- list includes them.+REVOKE ALL ON TABLE billing_meter.stripe_submission, billing_meter.stripe_submission_batch FROM PUBLIC;+REVOKE ALL ON FUNCTION billing_meter.enforce_stripe_submission_transition() FROM PUBLIC;+REVOKE ALL ON FUNCTION billing_meter.reject_stripe_submission_removal() FROM PUBLIC;++GRANT USAGE ON SCHEMA billing_meter TO cyclops_submitter;+GRANT SELECT ON TABLE billing_meter.reservation_hour_current, billing_meter.reservation_hour_collection_current TO cyclops_submitter;+GRANT SELECT, INSERT, UPDATE ON TABLE billing_meter.stripe_submission, billing_meter.stripe_submission_batch TO cyclops_submitter;+RESET ROLE;diff --git a/libs/fleet/backend/database/migrator.go b/libs/fleet/backend/database/migrator.goindex 9ec854e3b8..57dbfa006a 100644--- a/libs/fleet/backend/database/migrator.go+++ b/libs/fleet/backend/database/migrator.go@@ -55,6 +55,7 @@ type CredentialURLs struct {Metabase stringUsage stringMeter string+ Submitter string}type migrationFile struct {@@ -163,6 +164,7 @@ var expectedCredentialRoles = map[string]string{"metabase": "k8s_metabase","usage": "cyclops_usage_reader","meter": "cyclops_meter_writer",+ "submitter": "cyclops_submitter",}const selectAppliedMigrationsStatement = `select version, filename, sha256 from cyclops_migrations.applied_migrations order by application_order`@@ -484,6 +486,7 @@ func parseCredentialURLs(urls CredentialURLs) ([]credential, error) {{Name: "metabase", URL: urls.Metabase},{Name: "usage", URL: urls.Usage},{Name: "meter", URL: urls.Meter},+ {Name: "submitter", URL: urls.Submitter},}credentials := make([]credential, 0, len(inputs))@@ -519,6 +522,7 @@ func staticRoleContracts() []staticRoleContract {{role: "k8s_metabase", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},{role: "cyclops_usage_reader", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},{role: "cyclops_meter_writer", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},+ {role: "cyclops_submitter", login: true, connectionLimit: -1, validUntil: staticRoleValidUntilInfinity},}}@@ -795,6 +799,8 @@ func expectedReportingACLs() []reportingACL {{object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_fact"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},{object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_current"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},{object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "reservation_hour_collection_current"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},+ {object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "stripe_submission_batch"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},+ {object: reportingObject{kind: reportingObjectRelation, schema: "billing_meter", name: "stripe_submission"}, owner: "billing_meter_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "billing_meter_owner"},{object: reportingObject{kind: reportingObjectSchema, schema: "k8s_reporting"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeUsage, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_reporting", name: "current_resources"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},{object: reportingObject{kind: reportingObjectRelation, schema: "k8s_reporting", name: "hourly_reservation_usage"}, owner: "k8s_reporting_owner", privilege: reportingPrivilegeSelect, grantee: "k8s_metabase", grantor: "k8s_reporting_owner"},diff --git a/libs/fleet/backend/database/migrator_pg_test.go b/libs/fleet/backend/database/migrator_pg_test.goindex 391220d0c1..fff1c0de3f 100644--- a/libs/fleet/backend/database/migrator_pg_test.go+++ b/libs/fleet/backend/database/migrator_pg_test.go@@ -28,6 +28,7 @@ var staticMigrationRoles = []string{"cyclops_app","cyclops_usage_reader","cyclops_meter_writer",+ "cyclops_submitter","k8s_state_owner","k8s_state_writer","k8s_state_exporter",@@ -91,8 +92,8 @@ func TestInitialMigrationBuildsCompleteDatabase(t *testing.T) {firstSummary := captureRunSummary(t, func() error {return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})})- if firstSummary.Pending != 12 || firstSummary.Applied != 12 {- t.Fatalf("initial migration summary = %+v, want pending=12 applied=12", firstSummary)+ if firstSummary.Pending != 13 || firstSummary.Applied != 13 {+ t.Fatalf("initial migration summary = %+v, want pending=13 applied=13", firstSummary)}before := migrationLedgerRows(t, ctx, migrationURL)secondSummary := captureRunSummary(t, func() error {@@ -201,14 +202,14 @@ func TestRunUpgradesVersionOneAndThenNoOps(t *testing.T) {upgrade := captureRunSummary(t, func() error {return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})})- if upgrade.Current != 1 || upgrade.Target != 12 || upgrade.Pending != 11 || upgrade.Applied != 11 || upgrade.Skipped != 1 || upgrade.Result != "success" {+ if upgrade.Current != 1 || upgrade.Target != 13 || upgrade.Pending != 12 || upgrade.Applied != 12 || upgrade.Skipped != 1 || upgrade.Result != "success" {t.Fatalf("version-one upgrade summary = %+v", upgrade)}noOp := captureRunSummary(t, func() error {return Run(ctx, Config{MigrationURL: migrationURL, Credentials: credentials})})- if noOp.Current != 12 || noOp.Target != 12 || noOp.Pending != 0 || noOp.Applied != 0 || noOp.Skipped != 12 || noOp.Result != "success" {+ if noOp.Current != 13 || noOp.Target != 13 || noOp.Pending != 0 || noOp.Applied != 0 || noOp.Skipped != 13 || noOp.Result != "success" {t.Fatalf("post-upgrade no-op summary = %+v", noOp)}}@@ -1572,6 +1573,7 @@ func testCredentialURLs(t *testing.T, adminURL string) CredentialURLs {Metabase: withRole("k8s_metabase"),Usage: withRole("cyclops_usage_reader"),Meter: withRole("cyclops_meter_writer"),+ Submitter: withRole("cyclops_submitter"),}}@@ -1604,8 +1606,8 @@ func migrationLedgerRows(t *testing.T, ctx context.Context, adminURL string) []lif err := rows.Err(); err != nil {t.Fatal("iterate migration ledger")}- if len(ledger) != 12 {- t.Fatalf("migration ledger row count = %d, want 12", len(ledger))+ if len(ledger) != 13 {+ t.Fatalf("migration ledger row count = %d, want 13", len(ledger))}for index, want := range []struct {version int64@@ -1623,6 +1625,7 @@ func migrationLedgerRows(t *testing.T, ctx context.Context, adminURL string) []l{10, "000010_grant_metabase_billing_meter_access.sql"},{11, "000011_signed_service_urls.sql"},{12, "000012_private_account_lookup.sql"},+ {13, "000013_stripe_usage_submission.sql"},} {if ledger[index].Version != want.version || ledger[index].ApplicationOrder != int64(index+1) || ledger[index].Filename != want.filename {t.Fatalf("migration ledger row %d = version:%d order:%d filename:%q", index, ledger[index].Version, ledger[index].ApplicationOrder, ledger[index].Filename)@@ -1761,6 +1764,7 @@ func assertRoleContract(t *testing.T, ctx context.Context, adminURL string) {"k8s_metabase": {true, false, false},"cyclops_usage_reader": {true, false, false},"cyclops_meter_writer": {true, false, false},+ "cyclops_submitter": {true, false, false},}for role, want := range expected {var login, inherit, createRole, super, createDB, replication, bypassRLS bool@@ -2094,7 +2098,7 @@ func assertRuntimeLedgerAccess(t *testing.T, ctx context.Context, credentials Crvar count interr := connection.QueryRow(ctx, `select count(*) from cyclops_migrations.applied_migrations`).Scan(&count)connection.Close(ctx)- if err != nil || count != 12 {+ if err != nil || count != 13 {t.Errorf("%s ledger select = count:%d err:%v", role, count, err)}assertStatementFails(t, ctx, databaseURL, `insert into cyclops_migrations.applied_migrations (version, filename, sha256) values (99, 'invalid.sql', 'invalid')`)@@ -2590,6 +2594,8 @@ func assertExactReportingACLContract(t *testing.T, ctx context.Context, connecti('relation:billing_meter.reservation_hour_fact', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),('relation:billing_meter.reservation_hour_current', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),('relation:billing_meter.reservation_hour_collection_current', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),+ ('relation:billing_meter.stripe_submission_batch', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),+ ('relation:billing_meter.stripe_submission', 'SELECT', 'k8s_metabase', 'billing_meter_owner', false),('schema:k8s_reporting', 'USAGE', 'k8s_metabase', 'k8s_reporting_owner', false),('relation:k8s_reporting.current_resources', 'SELECT', 'k8s_metabase', 'k8s_reporting_owner', false),('relation:k8s_reporting.hourly_reservation_usage', 'SELECT', 'k8s_metabase', 'k8s_reporting_owner', false),diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.goindex 6775c96913..c033cb6175 100644--- a/libs/fleet/backend/database/migrator_test.go+++ b/libs/fleet/backend/database/migrator_test.go@@ -521,8 +521,8 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {if err != nil {t.Fatal(err)}- if len(files) != 12 {- t.Fatalf("expected exactly twelve migrations, got %d", len(files))+ if len(files) != 13 {+ t.Fatalf("expected exactly thirteen migrations, got %d", len(files))}manifest := make([]struct {Version int64@@ -550,6 +550,7 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {{10, "000010_grant_metabase_billing_meter_access.sql"},{11, "000011_signed_service_urls.sql"},{12, "000012_private_account_lookup.sql"},+ {13, "000013_stripe_usage_submission.sql"},}) {t.Fatalf("migration manifest = %#v", manifest)}@@ -927,12 +928,13 @@ func TestCredentialURLsExcludeDynamicTenantRoles(t *testing.T) {Metabase: "postgres://k8s_metabase:pw@db/cyclops",Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",+ Submitter: "postgres://cyclops_submitter:pw@db/cyclops",})if err != nil {t.Fatalf("parse fixed runtime credentials: %v", err)}- if len(credentials) != 7 {- t.Fatalf("credential count = %d, want 7", len(credentials))+ if len(credentials) != 8 {+ t.Fatalf("credential count = %d, want 8", len(credentials))}for _, credential := range credentials {if credential.Role == "k8s_query_broker" || strings.HasPrefix(credential.Role, "k8s_tenant_") {@@ -960,6 +962,7 @@ func TestStaticRoleContractsAreAllowlistedAndFixed(t *testing.T) {"k8s_metabase": {true, false, false, false, -1, staticRoleValidUntilInfinity},"cyclops_usage_reader": {true, false, false, false, -1, staticRoleValidUntilInfinity},"cyclops_meter_writer": {true, false, false, false, -1, staticRoleValidUntilInfinity},+ "cyclops_submitter": {true, false, false, false, -1, staticRoleValidUntilInfinity},}if len(contracts) != len(want) {t.Fatalf("static role contract count = %d, want %d", len(contracts), len(want))@@ -1010,6 +1013,7 @@ func TestCredentialURLsRequireExpectedRoleNames(t *testing.T) {Metabase: "postgres://k8s_metabase:pw@db/cyclops",Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",+ Submitter: "postgres://cyclops_submitter:pw@db/cyclops",})if err == nil {t.Fatal("expected application role-name validation")@@ -1028,6 +1032,7 @@ func TestCredentialURLsRejectEmptyPassword(t *testing.T) {Metabase: "postgres://k8s_metabase:pw@db/cyclops",Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",+ Submitter: "postgres://cyclops_submitter:pw@db/cyclops",})if err == nil || err.Error() != "application credential database URL must include a password" {t.Fatalf("expected empty password rejection, got %v", err)@@ -1043,6 +1048,7 @@ func TestCredentialURLParseErrorsPreserveCauseForBoundaryClassification(t *testiMetabase: "postgres://k8s_metabase:pw@db/cyclops",Usage: "postgres://cyclops_usage_reader:pw@db/cyclops",Meter: "postgres://cyclops_meter_writer:pw@db/cyclops",+ Submitter: "postgres://cyclops_submitter:pw@db/cyclops",})if !errors.Is(err, ErrInvalidConfiguration) {t.Fatalf("error = %v, want ErrInvalidConfiguration", err)diff --git a/libs/fleet/backend/database/test-setup.sh b/libs/fleet/backend/database/test-setup.shindex 10886bd802..518709f376 100755--- a/libs/fleet/backend/database/test-setup.sh+++ b/libs/fleet/backend/database/test-setup.sh@@ -26,6 +26,7 @@ static_roles=(k8s_metabasecyclops_usage_readercyclops_meter_writer+ cyclops_submitter)base_url="${admin_url%%\?*}"@@ -87,6 +88,7 @@ role_admin_url="$(url_for k8s_role_admin role-admin-password)"metabase_url="$(url_for k8s_metabase metabase-password)"usage_url="$(url_for cyclops_usage_reader usage-reader-password)"meter_url="$(url_for cyclops_meter_writer meter-writer-password)"+submitter_url="$(url_for cyclops_submitter submitter-password)"env_file="${CYCLOPS_TEST_DATABASE_ENV_FILE:-${GITHUB_ENV:-}}"if [[ -z "$env_file" ]]; then@@ -157,6 +159,7 @@ STATE_ROLE_ADMIN_DATABASE_URL="$role_admin_url" \METABASE_DATABASE_URL="$metabase_url" \USAGE_DATABASE_URL="$usage_url" \METER_DATABASE_URL="$meter_url" \+SUBMITTER_DATABASE_URL="$submitter_url" \go run ./cmd/db-migratecat >> "$env_file" <<EOF_ENV