Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
feat(billing): dry-run usage submitter binary (CUA-1158) (#7875)
ddupont592f6ee39e1d
diff --git a/libs/fleet/backend/Dockerfile b/libs/fleet/backend/Dockerfileindex 8632509957..fb59f66cfa 100644--- a/libs/fleet/backend/Dockerfile+++ b/libs/fleet/backend/Dockerfile@@ -12,7 +12,8 @@ RUN go install github.com/swaggo/swag/cmd/[email protected] && \RUN CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-cs-backend . \&& CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-db-migrate ./cmd/db-migrate \&& CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/account-lookup-backfill ./cmd/account-lookup-backfill \- && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-usage-meter ./cmd/usage-meter+ && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-usage-meter ./cmd/usage-meter \+ && CGO_ENABLED=1 GOOS=linux go build -trimpath -ldflags="-s -w" -o /out/cyclops-usage-submitter ./cmd/usage-submitterFROM --platform=linux/amd64 rust:1.94-bookworm AS rust-builderWORKDIR /src/k8s-state-query@@ -27,6 +28,7 @@ COPY --from=go-builder /out/cyclops-cs-backend /app/cyclops-cs-backendCOPY --from=go-builder /out/cyclops-db-migrate /app/cyclops-db-migrateCOPY --from=go-builder /out/account-lookup-backfill /app/account-lookup-backfillCOPY --from=go-builder /out/cyclops-usage-meter /app/cyclops-usage-meter+COPY --from=go-builder /out/cyclops-usage-submitter /app/cyclops-usage-submitterCOPY --from=rust-builder /out-k8s-state-projector /app/k8s-state-projectorUSER nonroot:nonrootEXPOSE 8080diff --git a/libs/fleet/backend/cmd/usage-submitter/main.go b/libs/fleet/backend/cmd/usage-submitter/main.gonew file mode 100644index 0000000000..5046b0c7e5--- /dev/null+++ b/libs/fleet/backend/cmd/usage-submitter/main.go@@ -0,0 +1,213 @@+// cyclops-usage-submitter turns completed reservation hours into Stripe+// meter events with a local ledger and an S3 archive written first.+//+// This build is dry-run only: SUBMIT_DRY_RUN must be "true" (the default).+// Design: docs/superpowers/specs/2026-09-16-cua-1158-stripe-usage-submitter-design.md+package main++import (+ "context"+ "errors"+ "fmt"+ "log/slog"+ "os"+ "strconv"+ "time"++ "cyclops-cs-backend/submitter"+ "cyclops-cs-backend/telemetry"+ "go.opentelemetry.io/otel"+ "go.opentelemetry.io/otel/attribute"+ "go.opentelemetry.io/otel/codes"+ "go.opentelemetry.io/otel/metric"+)++func main() {+ slog.SetDefault(slog.New(slog.NewJSONHandler(os.Stdout, nil)))+ if err := run(context.Background(), time.Now); err != nil {+ if errors.Is(err, submitter.ErrAlreadyRunning) {+ slog.Info("usage submitter skipped: another run holds the lock")+ return+ }+ slog.Error("usage submission failed", "error", err)+ os.Exit(1)+ }+}++func run(ctx context.Context, now func() time.Time) error {+ cfg, err := loadConfig()+ if err != nil {+ return err+ }+ shutdown, err := telemetry.Init(ctx, telemetry.Config{+ Endpoint: envDefault("OTEL_EXPORTER_OTLP_ENDPOINT", "https://otel.cua.ai"),+ Protocol: envDefault("OTEL_EXPORTER_OTLP_PROTOCOL", "http/protobuf"),+ ServiceName: envDefault("OTEL_SERVICE_NAME", "cyclops-usage-submitter"),+ ServiceNamespace: envDefault("OTEL_SERVICE_NAMESPACE", "cyclops-cs"),+ Environment: envDefault("OTEL_ENVIRONMENT", "production"),+ ResourceAttrs: os.Getenv("OTEL_RESOURCE_ATTRIBUTES"),+ })+ if err != nil {+ return fmt.Errorf("initialize telemetry: %w", err)+ }+ defer func() {+ shutdownCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second)+ defer cancel()+ if err := shutdown(shutdownCtx); err != nil {+ slog.Warn("telemetry shutdown failed")+ }+ }()++ ctx, span := otel.Tracer("cyclops-cs-backend/submitter").Start(ctx, "usage_submitter.run")+ defer span.End()+ span.SetAttributes(+ attribute.String("submitter.cluster_id", cfg.submitter.ClusterID),+ attribute.String("submitter.cutover_hour", cfg.submitter.CutoverHour.Format(time.RFC3339)),+ attribute.Bool("submitter.dry_run", cfg.submitter.DryRun),+ )++ store, err := submitter.NewPostgresStore(ctx, cfg.databaseURL)+ if err != nil {+ span.SetStatus(codes.Error, "database initialization failed")+ return err+ }+ defer store.Close()+ archiver, err := submitter.NewS3Archiver(ctx)+ if err != nil {+ span.SetStatus(codes.Error, "archive initialization failed")+ return err+ }++ started := now()+ result, runErr := submitter.Runner{+ Config: cfg.submitter,+ Store: store,+ Archiver: archiver,+ Customers: submitter.NoCustomers{},+ Now: now,+ }.Run(ctx)++ meter := otel.Meter("cyclops-cs-backend/submitter")+ duration, _ := meter.Float64Histogram("cyclops.usage_submitter.run.duration", metric.WithDescription("Usage submitter run duration."), metric.WithUnit("s"))+ duration.Record(ctx, now().Sub(started).Seconds())+ if runErr != nil {+ if !errors.Is(runErr, submitter.ErrAlreadyRunning) {+ failures, _ := meter.Int64Counter("cyclops.usage_submitter.run.failures")+ failures.Add(ctx, 1)+ span.SetStatus(codes.Error, "usage submission failed")+ }+ return runErr+ }+ submissions, _ := meter.Int64Counter("cyclops.usage_submitter.submissions")+ submissions.Add(ctx, int64(result.Created), outcome("created"))+ submissions.Add(ctx, int64(result.Unchanged), outcome("unchanged"))+ submissions.Add(ctx, int64(result.Superseded), outcome("superseded"))+ submissions.Add(ctx, int64(result.DryRun), outcome("dry_run"))+ submissions.Add(ctx, int64(result.Unmapped), outcome("unmapped"))+ age, _ := meter.Float64Gauge("cyclops.usage_submitter.oldest_unsent_hour_age", metric.WithDescription("Age of the oldest eligible tenant-hour without a terminal submission."), metric.WithUnit("s"))+ if !result.OldestUnsentHour.IsZero() {+ age.Record(ctx, now().Sub(result.OldestUnsentHour).Seconds())+ } else {+ age.Record(ctx, 0)+ }+ span.SetAttributes(+ attribute.String("submitter.batch_id", result.BatchID.String()),+ attribute.Int("submitter.hours_considered", result.HoursConsidered),+ attribute.Int("submitter.tenant_hours", result.TenantHours),+ attribute.Int("submitter.created", result.Created),+ attribute.Int("submitter.unchanged", result.Unchanged),+ attribute.Int("submitter.resumed_batches", result.ResumedBatches),+ )+ slog.Info("usage submission complete",+ "batch_id", result.BatchID.String(),+ "dry_run", cfg.submitter.DryRun,+ "hours_considered", result.HoursConsidered,+ "tenant_hours", result.TenantHours,+ "created", result.Created,+ "unchanged", result.Unchanged,+ "superseded", result.Superseded,+ "dry_run_marked", result.DryRun,+ "resumed_batches", result.ResumedBatches)+ return nil+}++type config struct {+ databaseURL string+ submitter submitter.Config+}++func loadConfig() (config, error) {+ cfg := config{+ databaseURL: os.Getenv("SUBMIT_DATABASE_URL"),+ submitter: submitter.Config{+ ClusterID: envDefault("SUBMIT_CLUSTER_ID", "kopf-k3s"),+ SettlementLag: 2 * time.Hour,+ Lookback: 34 * 24 * time.Hour,+ DryRun: true,+ ArchiveBucket: os.Getenv("SUBMIT_ARCHIVE_BUCKET"),+ ArchivePrefix: os.Getenv("SUBMIT_ARCHIVE_PREFIX"),+ ExporterVersion: envDefault("SUBMIT_EXPORTER_VERSION", "dev"),+ },+ }+ if cfg.databaseURL == "" {+ return config{}, errors.New("SUBMIT_DATABASE_URL is required")+ }+ if cfg.submitter.ArchiveBucket == "" {+ return config{}, errors.New("SUBMIT_ARCHIVE_BUCKET is required")+ }+ rawCutover := os.Getenv("SUBMIT_CUTOVER_HOUR")+ if rawCutover == "" {+ return config{}, errors.New("SUBMIT_CUTOVER_HOUR is required and has no default: it is the manual-to-automated billing boundary")+ }+ cutover, err := time.Parse(time.RFC3339, rawCutover)+ if err != nil {+ return config{}, fmt.Errorf("SUBMIT_CUTOVER_HOUR must be an exact UTC RFC3339 hour: %w", err)+ }+ cfg.submitter.CutoverHour = cutover.UTC()+ if !cfg.submitter.CutoverHour.Equal(cfg.submitter.CutoverHour.Truncate(time.Hour)) {+ return config{}, errors.New("SUBMIT_CUTOVER_HOUR must be an exact UTC RFC3339 hour")+ }+ if cfg.submitter.SettlementLag, err = durationEnv("SUBMIT_SETTLEMENT_LAG", cfg.submitter.SettlementLag); err != nil {+ return config{}, err+ }+ if cfg.submitter.Lookback, err = durationEnv("SUBMIT_LOOKBACK", cfg.submitter.Lookback); err != nil {+ return config{}, err+ }+ if raw := os.Getenv("SUBMIT_DRY_RUN"); raw != "" {+ dryRun, err := strconv.ParseBool(raw)+ if err != nil {+ return config{}, fmt.Errorf("SUBMIT_DRY_RUN must be a boolean: %w", err)+ }+ cfg.submitter.DryRun = dryRun+ }+ if !cfg.submitter.DryRun {+ return config{}, errors.New("SUBMIT_DRY_RUN=false is not supported by this build; live submission is not implemented")+ }+ return cfg, nil+}++func durationEnv(name string, fallback time.Duration) (time.Duration, error) {+ raw := os.Getenv(name)+ if raw == "" {+ return fallback, nil+ }+ value, err := time.ParseDuration(raw)+ if err != nil {+ return 0, fmt.Errorf("%s must be a positive duration: %w", name, err)+ }+ if value <= 0 {+ return 0, fmt.Errorf("%s must be a positive duration", name)+ }+ return value, nil+}++func envDefault(name, fallback string) string {+ if value := os.Getenv(name); value != "" {+ return value+ }+ return fallback+}++func outcome(value string) metric.AddOption {+ return metric.WithAttributes(attribute.String("outcome", value))+}diff --git a/libs/fleet/backend/submitter/archive.go b/libs/fleet/backend/submitter/archive.gonew file mode 100644index 0000000000..65ad1feaf6--- /dev/null+++ b/libs/fleet/backend/submitter/archive.go@@ -0,0 +1,135 @@+package submitter++import (+ "bytes"+ "context"+ "crypto/sha256"+ "encoding/hex"+ "encoding/json"+ "errors"+ "fmt"+ "sort"+ "time"++ awsconfig "github.com/aws/aws-sdk-go-v2/config"+ "github.com/aws/aws-sdk-go-v2/service/s3"+)++// Archive is the S3 representation of one batch: newline-delimited JSON rows+// plus a manifest written last, mirroring the opencost-pipeline export+// contract (manifest presence means the data object is complete).+type Archive struct {+ Data []byte+ Manifest []byte+ SHA256 string+}++type archiveRow struct {+ SubmissionID string `json:"submission_id"`+ BatchID string `json:"batch_id"`+ CapsuleTenant string `json:"capsule_tenant"`+ HourStart string `json:"hour_start"`+ Meter string `json:"meter"`+ Seq int `json:"seq"`+ Identifier string `json:"identifier"`+ StripeCustomerID string `json:"stripe_customer_id,omitempty"`+ Quantity string `json:"quantity"`+ FactSetSHA256 string `json:"fact_set_sha256"`+}++type archiveManifest struct {+ SchemaVersion string `json:"schema_version"`+ Dataset string `json:"dataset"`+ BatchID string `json:"batch_id"`+ Cluster string `json:"cluster"`+ CutoverHour string `json:"cutover_hour"`+ WindowStart string `json:"window_start"`+ WindowEnd string `json:"window_end"`+ DryRun bool `json:"dry_run"`+ ExporterVersion string `json:"exporter_version"`+ ExportedAt string `json:"exported_at"`+ DataKey string `json:"data_key"`+ DataSHA256 string `json:"data_sha256"`+ RowCount int `json:"row_count"`+}++// BuildArchive renders rows deterministically (sorted by identifier) so a+// resumed batch produces byte-identical objects.+func BuildArchive(batch Batch, rows []Submission, exporterVersion string, exportedAt time.Time) (Archive, error) {+ if len(rows) == 0 {+ return Archive{}, errors.New("archive requires at least one submission")+ }+ sorted := make([]Submission, len(rows))+ copy(sorted, rows)+ sort.Slice(sorted, func(i, j int) bool { return sorted[i].Identifier < sorted[j].Identifier })++ var data bytes.Buffer+ encoder := json.NewEncoder(&data)+ for _, row := range sorted {+ if row.BatchID != batch.BatchID {+ return Archive{}, fmt.Errorf("submission %s belongs to batch %s, not %s", row.Identifier, row.BatchID, batch.BatchID)+ }+ if err := encoder.Encode(archiveRow{+ SubmissionID: row.SubmissionID.String(),+ BatchID: row.BatchID.String(),+ CapsuleTenant: row.Tenant,+ HourStart: row.HourStart.UTC().Format(time.RFC3339),+ Meter: row.Meter,+ Seq: row.Seq,+ Identifier: row.Identifier,+ StripeCustomerID: row.StripeCustomerID,+ Quantity: row.Quantity,+ FactSetSHA256: row.FactSetSHA256,+ }); err != nil {+ return Archive{}, fmt.Errorf("encode archive row %s: %w", row.Identifier, err)+ }+ }+ digest := sha256.Sum256(data.Bytes())+ sha := hex.EncodeToString(digest[:])++ manifest, err := json.Marshal(archiveManifest{+ SchemaVersion: SchemaVersion,+ Dataset: ArchiveDataset,+ BatchID: batch.BatchID.String(),+ Cluster: batch.ClusterID,+ CutoverHour: batch.CutoverHour.UTC().Format(time.RFC3339),+ WindowStart: batch.WindowStart.UTC().Format(time.RFC3339),+ WindowEnd: batch.WindowEnd.UTC().Format(time.RFC3339),+ DryRun: batch.DryRun,+ ExporterVersion: exporterVersion,+ ExportedAt: exportedAt.UTC().Format(time.RFC3339),+ DataKey: batch.ArchiveKey,+ DataSHA256: sha,+ RowCount: len(sorted),+ })+ if err != nil {+ return Archive{}, fmt.Errorf("encode archive manifest: %w", err)+ }+ return Archive{Data: data.Bytes(), Manifest: append(manifest, '\n'), SHA256: sha}, nil+}++// S3Archiver writes archive objects with the ambient AWS credential chain+// (web identity in the CronJob).+type S3Archiver struct {+ client *s3.Client+}++func NewS3Archiver(ctx context.Context) (*S3Archiver, error) {+ configuration, err := awsconfig.LoadDefaultConfig(ctx)+ if err != nil {+ return nil, fmt.Errorf("load AWS configuration: %w", err)+ }+ return &S3Archiver{client: s3.NewFromConfig(configuration)}, nil+}++func (a *S3Archiver) Put(ctx context.Context, bucket, key, contentType string, body []byte) error {+ if _, err := a.client.PutObject(ctx, &s3.PutObjectInput{+ Bucket: &bucket,+ Key: &key,+ ContentType: &contentType,+ Body: bytes.NewReader(body),+ }); err != nil {+ return fmt.Errorf("put s3://%s/%s: %w", bucket, key, err)+ }+ return nil+}diff --git a/libs/fleet/backend/submitter/postgres.go b/libs/fleet/backend/submitter/postgres.gonew file mode 100644index 0000000000..dc789a6e3f--- /dev/null+++ b/libs/fleet/backend/submitter/postgres.go@@ -0,0 +1,378 @@+package submitter++import (+ "context"+ "errors"+ "fmt"+ "time"++ "github.com/google/uuid"+ "github.com/jackc/pgx/v5"+ "github.com/jackc/pgx/v5/pgxpool"+)++const submitterRole = "cyclops_submitter"++const submitterLockKey = "cyclops-usage-submitter"++const tryLockStatement = `select pg_try_advisory_lock(hashtextextended($1, 0))`++const unlockStatement = `select pg_advisory_unlock(hashtextextended($1, 0))`++const latestLiveCutoverStatement = `select cutover_hour+ from billing_meter.stripe_submission_batch+ where dry_run = false+ order by cutover_hour desc+ limit 1`++const completedHoursStatement = `select hour_start+ from billing_meter.reservation_hour_collection_current+ where cluster_id = $1 and hour_start >= $2 and hour_start < $3+ order by hour_start`++// Quantities are computed in Postgres numeric and truncated once, per the+// CUA-1150 contract: one unit is 1 vCPU or 1 GiB (2^30 bytes) for 3600 s.+const aggregateTenantHoursStatement = `select+ capsule_tenant,+ hour_start,+ trunc(sum(virtual_cpu_core_seconds) / 3600, 6)::text,+ trunc(sum(virtual_memory_byte_seconds) / (1073741824::numeric * 3600), 6)::text,+ array_agg(fact_id::text order by fact_id),+ array_agg(revision order by fact_id),+ array_agg(virtual_cpu_core_seconds::text order by fact_id),+ array_agg(virtual_memory_byte_seconds::text order by fact_id)+ from billing_meter.reservation_hour_current+ where cluster_id = $1 and hour_start = any($2)+ group by capsule_tenant, hour_start+ order by capsule_tenant, hour_start`++const ledgerHeadsStatement = `select distinct on (capsule_tenant, hour_start, meter)+ submission_id, batch_id, capsule_tenant, hour_start, meter, seq, status, fact_set_sha256+ from billing_meter.stripe_submission+ where hour_start = any($1)+ order by capsule_tenant, hour_start, meter, seq desc`++const insertBatchStatement = `insert into billing_meter.stripe_submission_batch (+ batch_id, cluster_id, cutover_hour, window_start, window_end, dry_run,+ archive_bucket, archive_key, submission_count, created_at+) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10)`++const lockTenantStatement = `select pg_advisory_xact_lock(hashtextextended($1, 0))`++const nextSeqStatement = `select coalesce(max(seq), 0) + 1+ from billing_meter.stripe_submission+ where capsule_tenant = $1 and hour_start = $2 and meter = $3`++const insertSubmissionStatement = `insert into billing_meter.stripe_submission (+ submission_id, batch_id, capsule_tenant, hour_start, meter, seq, identifier,+ stripe_customer_id, quantity, fact_set_sha256, status, created_at+) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`++const incompleteBatchesStatement = `select+ batch_id, cluster_id, cutover_hour, window_start, window_end, dry_run,+ archive_bucket, archive_key, coalesce(archive_sha256, ''), submission_count,+ created_at, archived_at, completed_at+ from billing_meter.stripe_submission_batch+ where completed_at is null+ order by created_at`++const batchSubmissionsStatement = `select+ submission_id, batch_id, capsule_tenant, hour_start, meter, seq, identifier,+ coalesce(stripe_customer_id, ''), quantity::text, fact_set_sha256, status, created_at+ from billing_meter.stripe_submission+ where batch_id = $1+ order by identifier`++const markBatchArchivedStatement = `update billing_meter.stripe_submission_batch+ set archive_sha256 = $2, archived_at = $3+ where batch_id = $1 and archived_at is null`++const markBatchCompletedStatement = `update billing_meter.stripe_submission_batch+ set completed_at = $2+ where batch_id = $1 and completed_at is null`++const markSubmissionStatement = `update billing_meter.stripe_submission+ set status = $2, stripe_event_id = nullif($3, ''), error_code = nullif($4, ''),+ error_message = nullif($5, ''), sent_at = $6+ where submission_id = $1 and status = 'archived'`++type PostgresStore struct {+ pool *pgxpool.Pool+}++func NewPostgresStore(ctx context.Context, databaseURL string) (*PostgresStore, error) {+ config, err := pgxpool.ParseConfig(databaseURL)+ if err != nil {+ return nil, fmt.Errorf("parse submitter database URL: %w", err)+ }+ if config.ConnConfig.User != submitterRole {+ return nil, fmt.Errorf("submitter database URL must use %s", submitterRole)+ }+ config.MaxConns = 2+ config.MinConns = 0+ config.MaxConnIdleTime = time.Minute+ config.MaxConnLifetime = 15 * time.Minute+ pool, err := pgxpool.NewWithConfig(ctx, config)+ if err != nil {+ return nil, fmt.Errorf("open submitter database pool: %w", err)+ }+ return &PostgresStore{pool: pool}, nil+}++func (s *PostgresStore) Close() {+ if s != nil && s.pool != nil {+ s.pool.Close()+ }+}++// TryLock holds a session advisory lock on a dedicated connection for the+// life of the run, so two overlapping runs cannot both create batches.+func (s *PostgresStore) TryLock(ctx context.Context) (bool, func(context.Context) error, error) {+ connection, err := s.pool.Acquire(ctx)+ if err != nil {+ return false, nil, fmt.Errorf("acquire lock connection: %w", err)+ }+ var locked bool+ if err := connection.QueryRow(ctx, tryLockStatement, submitterLockKey).Scan(&locked); err != nil {+ connection.Release()+ return false, nil, fmt.Errorf("try submitter lock: %w", err)+ }+ if !locked {+ connection.Release()+ return false, func(context.Context) error { return nil }, nil+ }+ unlock := func(ctx context.Context) error {+ defer connection.Release()+ var released bool+ if err := connection.QueryRow(ctx, unlockStatement, submitterLockKey).Scan(&released); err != nil {+ return fmt.Errorf("release submitter lock: %w", err)+ }+ if !released {+ return errors.New("submitter lock was not held at release")+ }+ return nil+ }+ return true, unlock, nil+}++func (s *PostgresStore) LatestLiveCutover(ctx context.Context) (time.Time, bool, error) {+ var cutover time.Time+ err := s.pool.QueryRow(ctx, latestLiveCutoverStatement).Scan(&cutover)+ if errors.Is(err, pgx.ErrNoRows) {+ return time.Time{}, false, nil+ }+ if err != nil {+ return time.Time{}, false, fmt.Errorf("query live cutover: %w", err)+ }+ return cutover.UTC(), true, nil+}++func (s *PostgresStore) CompletedHours(ctx context.Context, clusterID string, from, to time.Time) ([]time.Time, error) {+ rows, err := s.pool.Query(ctx, completedHoursStatement, clusterID, from, to)+ if err != nil {+ return nil, fmt.Errorf("query completed hours: %w", err)+ }+ defer rows.Close()+ var hours []time.Time+ for rows.Next() {+ var hour time.Time+ if err := rows.Scan(&hour); err != nil {+ return nil, fmt.Errorf("scan completed hour: %w", err)+ }+ hours = append(hours, hour.UTC())+ }+ if err := rows.Err(); err != nil {+ return nil, fmt.Errorf("iterate completed hours: %w", err)+ }+ return hours, nil+}++func (s *PostgresStore) AggregateTenantHours(ctx context.Context, clusterID string, hours []time.Time) ([]TenantHour, error) {+ rows, err := s.pool.Query(ctx, aggregateTenantHoursStatement, clusterID, hours)+ if err != nil {+ return nil, fmt.Errorf("query tenant hours: %w", err)+ }+ defer rows.Close()+ var result []TenantHour+ for rows.Next() {+ var tenantHour TenantHour+ var factIDs, cpuSeconds, memorySeconds []string+ var revisions []int32+ if err := rows.Scan(&tenantHour.Tenant, &tenantHour.HourStart, &tenantHour.VCPUHours, &tenantHour.GiBHours, &factIDs, &revisions, &cpuSeconds, &memorySeconds); err != nil {+ return nil, fmt.Errorf("scan tenant hour: %w", err)+ }+ if len(factIDs) != len(revisions) || len(factIDs) != len(cpuSeconds) || len(factIDs) != len(memorySeconds) {+ return nil, fmt.Errorf("tenant hour %s/%s returned mismatched fact arrays", tenantHour.Tenant, tenantHour.HourStart.Format(time.RFC3339))+ }+ tenantHour.HourStart = tenantHour.HourStart.UTC()+ for index := range factIDs {+ tenantHour.Facts = append(tenantHour.Facts, FactRef{+ FactID: factIDs[index],+ Revision: int(revisions[index]),+ CPUCoreSeconds: cpuSeconds[index],+ MemoryByteSeconds: memorySeconds[index],+ })+ }+ result = append(result, tenantHour)+ }+ if err := rows.Err(); err != nil {+ return nil, fmt.Errorf("iterate tenant hours: %w", err)+ }+ return result, nil+}++func (s *PostgresStore) LedgerHeads(ctx context.Context, hours []time.Time) (map[LedgerKey]LedgerHead, error) {+ rows, err := s.pool.Query(ctx, ledgerHeadsStatement, hours)+ if err != nil {+ return nil, fmt.Errorf("query ledger heads: %w", err)+ }+ defer rows.Close()+ heads := map[LedgerKey]LedgerHead{}+ for rows.Next() {+ var key LedgerKey+ var head LedgerHead+ if err := rows.Scan(&head.SubmissionID, &head.BatchID, &key.Tenant, &key.HourStart, &key.Meter, &head.Seq, &head.Status, &head.FactSetSHA256); err != nil {+ return nil, fmt.Errorf("scan ledger head: %w", err)+ }+ key.HourStart = key.HourStart.UTC()+ heads[key] = head+ }+ if err := rows.Err(); err != nil {+ return nil, fmt.Errorf("iterate ledger heads: %w", err)+ }+ return heads, nil+}++// CreateBatch inserts the batch and its submissions in one transaction. Seq+// values are re-derived under a per-tenant advisory lock so a concurrent+// writer (which the run lock should already prevent) cannot collide.+func (s *PostgresStore) CreateBatch(ctx context.Context, batch Batch, submissions []Submission) ([]Submission, error) {+ transaction, err := s.pool.BeginTx(ctx, pgx.TxOptions{})+ if err != nil {+ return nil, fmt.Errorf("begin batch transaction: %w", err)+ }+ defer func() { _ = transaction.Rollback(ctx) }()++ if _, err := transaction.Exec(ctx, insertBatchStatement,+ batch.BatchID, batch.ClusterID, batch.CutoverHour, batch.WindowStart, batch.WindowEnd, batch.DryRun,+ batch.ArchiveBucket, batch.ArchiveKey, len(submissions), batch.CreatedAt); err != nil {+ return nil, fmt.Errorf("insert batch: %w", err)+ }+ created := make([]Submission, 0, len(submissions))+ for _, submission := range submissions {+ if _, err := transaction.Exec(ctx, lockTenantStatement, submission.Tenant); err != nil {+ return nil, fmt.Errorf("lock tenant %q: %w", submission.Tenant, err)+ }+ var seq int+ if err := transaction.QueryRow(ctx, nextSeqStatement, submission.Tenant, submission.HourStart, submission.Meter).Scan(&seq); err != nil {+ return nil, fmt.Errorf("read next seq for %s: %w", submission.Identifier, err)+ }+ if seq != submission.Seq {+ return nil, fmt.Errorf("seq for %s changed underneath the run: planned %d, ledger says %d", submission.Identifier, submission.Seq, seq)+ }+ var customer any+ if submission.StripeCustomerID != "" {+ customer = submission.StripeCustomerID+ }+ if _, err := transaction.Exec(ctx, insertSubmissionStatement,+ submission.SubmissionID, submission.BatchID, submission.Tenant, submission.HourStart, submission.Meter,+ submission.Seq, submission.Identifier, customer, submission.Quantity, submission.FactSetSHA256,+ submission.Status, submission.CreatedAt); err != nil {+ return nil, fmt.Errorf("insert submission %s: %w", submission.Identifier, err)+ }+ created = append(created, submission)+ }+ if err := transaction.Commit(ctx); err != nil {+ return nil, fmt.Errorf("commit batch: %w", err)+ }+ return created, nil+}++func (s *PostgresStore) IncompleteBatches(ctx context.Context) ([]Batch, error) {+ rows, err := s.pool.Query(ctx, incompleteBatchesStatement)+ if err != nil {+ return nil, fmt.Errorf("query incomplete batches: %w", err)+ }+ defer rows.Close()+ var batches []Batch+ for rows.Next() {+ var batch Batch+ var archivedAt, completedAt *time.Time+ if err := rows.Scan(&batch.BatchID, &batch.ClusterID, &batch.CutoverHour, &batch.WindowStart, &batch.WindowEnd, &batch.DryRun,+ &batch.ArchiveBucket, &batch.ArchiveKey, &batch.ArchiveSHA256, &batch.SubmissionCount,+ &batch.CreatedAt, &archivedAt, &completedAt); err != nil {+ return nil, fmt.Errorf("scan incomplete batch: %w", err)+ }+ if archivedAt != nil {+ batch.ArchivedAt = archivedAt.UTC()+ }+ if completedAt != nil {+ batch.CompletedAt = completedAt.UTC()+ }+ batches = append(batches, batch)+ }+ if err := rows.Err(); err != nil {+ return nil, fmt.Errorf("iterate incomplete batches: %w", err)+ }+ return batches, nil+}++func (s *PostgresStore) BatchSubmissions(ctx context.Context, batchID uuid.UUID) ([]Submission, error) {+ rows, err := s.pool.Query(ctx, batchSubmissionsStatement, batchID)+ if err != nil {+ return nil, fmt.Errorf("query batch submissions: %w", err)+ }+ defer rows.Close()+ var submissions []Submission+ for rows.Next() {+ var submission Submission+ if err := rows.Scan(&submission.SubmissionID, &submission.BatchID, &submission.Tenant, &submission.HourStart, &submission.Meter,+ &submission.Seq, &submission.Identifier, &submission.StripeCustomerID, &submission.Quantity, &submission.FactSetSHA256,+ &submission.Status, &submission.CreatedAt); err != nil {+ return nil, fmt.Errorf("scan batch submission: %w", err)+ }+ submission.HourStart = submission.HourStart.UTC()+ submissions = append(submissions, submission)+ }+ if err := rows.Err(); err != nil {+ return nil, fmt.Errorf("iterate batch submissions: %w", err)+ }+ return submissions, nil+}++func (s *PostgresStore) MarkBatchArchived(ctx context.Context, batchID uuid.UUID, sha256 string, at time.Time) error {+ tag, err := s.pool.Exec(ctx, markBatchArchivedStatement, batchID, sha256, at)+ if err != nil {+ return fmt.Errorf("mark batch %s archived: %w", batchID, err)+ }+ if tag.RowsAffected() != 1 {+ return fmt.Errorf("batch %s was not in the unarchived state", batchID)+ }+ return nil+}++func (s *PostgresStore) MarkBatchCompleted(ctx context.Context, batchID uuid.UUID, at time.Time) error {+ tag, err := s.pool.Exec(ctx, markBatchCompletedStatement, batchID, at)+ if err != nil {+ return fmt.Errorf("mark batch %s completed: %w", batchID, err)+ }+ if tag.RowsAffected() != 1 {+ return fmt.Errorf("batch %s was not in the incomplete state", batchID)+ }+ return nil+}++func (s *PostgresStore) MarkSubmission(ctx context.Context, submissionID uuid.UUID, outcome Outcome) error {+ var sentAt any+ if !outcome.SentAt.IsZero() {+ sentAt = outcome.SentAt+ }+ tag, err := s.pool.Exec(ctx, markSubmissionStatement, submissionID, outcome.Status, outcome.StripeEventID, outcome.ErrorCode, outcome.ErrorMessage, sentAt)+ if err != nil {+ return fmt.Errorf("mark submission %s %s: %w", submissionID, outcome.Status, err)+ }+ if tag.RowsAffected() != 1 {+ return fmt.Errorf("submission %s was not in the archived state", submissionID)+ }+ return nil+}diff --git a/libs/fleet/backend/submitter/postgres_test.go b/libs/fleet/backend/submitter/postgres_test.gonew file mode 100644index 0000000000..9f231aa92c--- /dev/null+++ b/libs/fleet/backend/submitter/postgres_test.go@@ -0,0 +1,39 @@+package submitter++import (+ "strings"+ "testing"+)++func TestStatementsRetainContractParameters(t *testing.T) {+ for name, contract := range map[string]struct {+ statement string+ expect []string+ }{+ "aggregate": {aggregateTenantHoursStatement, []string{+ "trunc(sum(virtual_cpu_core_seconds) / 3600, 6)",+ "trunc(sum(virtual_memory_byte_seconds) / (1073741824::numeric * 3600), 6)",+ "billing_meter.reservation_hour_current",+ "group by capsule_tenant, hour_start",+ }},+ "completed hours": {completedHoursStatement, []string{"billing_meter.reservation_hour_collection_current", "cluster_id = $1"}},+ "ledger heads": {ledgerHeadsStatement, []string{"distinct on (capsule_tenant, hour_start, meter)", "seq desc"}},+ "next seq": {nextSeqStatement, []string{"coalesce(max(seq), 0) + 1"}},+ "mark submission": {markSubmissionStatement, []string{"where submission_id = $1 and status = 'archived'"}},+ "mark archived": {markBatchArchivedStatement, []string{"archived_at is null"}},+ "mark completed": {markBatchCompletedStatement, []string{"completed_at is null"}},+ "live cutover": {latestLiveCutoverStatement, []string{"dry_run = false"}},+ } {+ for _, expected := range contract.expect {+ if !strings.Contains(contract.statement, expected) {+ t.Errorf("%s statement is missing %q", name, expected)+ }+ }+ }+}++func TestNewPostgresStoreRequiresSubmitterRole(t *testing.T) {+ if _, err := NewPostgresStore(t.Context(), "postgres://cyclops_meter_writer:pw@localhost/cyclops"); err == nil || !strings.Contains(err.Error(), submitterRole) {+ t.Fatalf("expected role validation error, got %v", err)+ }+}diff --git a/libs/fleet/backend/submitter/submitter.go b/libs/fleet/backend/submitter/submitter.gonew file mode 100644index 0000000000..96fe314b60--- /dev/null+++ b/libs/fleet/backend/submitter/submitter.go@@ -0,0 +1,494 @@+// Package submitter turns completed reservation hours in billing_meter into+// Stripe meter events, exactly once per tenant-hour-meter, with a durable+// local ledger and an S3 archive written before anything is sent.+//+// Design: docs/superpowers/specs/2026-09-16-cua-1158-stripe-usage-submitter-design.md+//+// This package currently implements the dry-run path only: it selects,+// aggregates, ledgers, and archives, then marks rows dry_run. The Stripe+// path is added behind DryRun=false in a later change.+package submitter++import (+ "context"+ "crypto/sha256"+ "encoding/hex"+ "errors"+ "fmt"+ "sort"+ "strings"+ "time"++ "github.com/google/uuid"+)++const (+ MeterVCPUHours = "cua_vcpu_hours"+ MeterGiBHours = "cua_gib_hours"++ StatusArchived = "archived"+ StatusSent = "sent"+ StatusDryRun = "dry_run"+ StatusFailed = "failed"+ StatusOutOfWindow = "out_of_window"+ StatusUnmapped = "unmapped"++ // ErrorCodeSuperseded marks an in-flight row whose facts changed before it+ // was completed; a newer seq carries the recomputed quantity.+ ErrorCodeSuperseded = "superseded"++ // MaxIdentifierLength is Stripe's limit on meter event identifiers.+ MaxIdentifierLength = 100++ ArchiveDataset = "reservation-usage"+ SchemaVersion = "v1"+)++var Meters = []string{MeterVCPUHours, MeterGiBHours}++type Config struct {+ ClusterID string+ CutoverHour time.Time+ SettlementLag time.Duration+ Lookback time.Duration+ DryRun bool+ ArchiveBucket string+ ArchivePrefix string+ ExporterVersion string+}++func (c Config) validate() error {+ if strings.TrimSpace(c.ClusterID) == "" {+ return errors.New("cluster id is required")+ }+ if c.CutoverHour.IsZero() || !c.CutoverHour.Equal(c.CutoverHour.UTC().Truncate(time.Hour)) {+ return errors.New("cutover hour must be an exact UTC hour")+ }+ if c.SettlementLag <= 0 || c.Lookback <= 0 {+ return errors.New("settlement lag and lookback must be positive")+ }+ if strings.TrimSpace(c.ArchiveBucket) == "" {+ return errors.New("archive bucket is required")+ }+ if !c.DryRun {+ return errors.New("live submission is not implemented; DryRun must be true")+ }+ return nil+}++// FactRef is one current-revision fact row contributing to a tenant-hour.+type FactRef struct {+ FactID string+ Revision int+ CPUCoreSeconds string+ MemoryByteSeconds string+}++// TenantHour is the aggregate the submitter bills: one tenant, one UTC hour,+// both meters. Quantities are decimal strings already truncated to six+// places by Postgres numeric arithmetic.+type TenantHour struct {+ Tenant string+ HourStart time.Time+ VCPUHours string+ GiBHours string+ Facts []FactRef+}++func (h TenantHour) Quantity(meter string) (string, error) {+ switch meter {+ case MeterVCPUHours:+ return h.VCPUHours, nil+ case MeterGiBHours:+ return h.GiBHours, nil+ }+ return "", fmt.Errorf("unknown meter %q", meter)+}++// FactSetSHA256 is order-independent over (fact_id, value) for the meter's+// own dimension. The revision is deliberately excluded: a revision that+// changes only CPU seconds must not resubmit an identical memory quantity.+func FactSetSHA256(meter string, facts []FactRef) (string, error) {+ lines := make([]string, 0, len(facts))+ for _, fact := range facts {+ value := fact.CPUCoreSeconds+ switch meter {+ case MeterVCPUHours:+ case MeterGiBHours:+ value = fact.MemoryByteSeconds+ default:+ return "", fmt.Errorf("unknown meter %q", meter)+ }+ lines = append(lines, fact.FactID+":"+value)+ }+ sort.Strings(lines)+ digest := sha256.Sum256([]byte(strings.Join(lines, "\n")))+ return hex.EncodeToString(digest[:]), nil+}++// Identifier is the Stripe meter event identifier: deterministic per+// tenant-hour-meter-seq so retries dedupe and a new seq overwrites the hour.+func Identifier(meter, tenant string, hourStart time.Time, seq int) (string, error) {+ if seq <= 0 {+ return "", errors.New("seq must be positive")+ }+ identifier := fmt.Sprintf("%s:%s:%s:r%d", meter, tenant, hourStart.UTC().Format(time.RFC3339), seq)+ if len(identifier) > MaxIdentifierLength {+ return "", fmt.Errorf("identifier for tenant %q exceeds %d characters", tenant, MaxIdentifierLength)+ }+ return identifier, nil+}++// Submission mirrors a billing_meter.stripe_submission row.+type Submission struct {+ SubmissionID uuid.UUID+ BatchID uuid.UUID+ Tenant string+ HourStart time.Time+ Meter string+ Seq int+ Identifier string+ StripeCustomerID string+ Quantity string+ FactSetSHA256 string+ Status string+ CreatedAt time.Time+}++// LedgerHead is the latest row for a tenant-hour-meter.+type LedgerHead struct {+ SubmissionID uuid.UUID+ BatchID uuid.UUID+ Seq int+ Status string+ FactSetSHA256 string+}++type LedgerKey struct {+ Tenant string+ HourStart time.Time+ Meter string+}++type Batch struct {+ BatchID uuid.UUID+ ClusterID string+ CutoverHour time.Time+ WindowStart time.Time+ WindowEnd time.Time+ DryRun bool+ ArchiveBucket string+ ArchiveKey string+ ArchiveSHA256 string+ SubmissionCount int+ CreatedAt time.Time+ ArchivedAt time.Time+ CompletedAt time.Time+}++// Outcome is a terminal status applied to an archived submission.+type Outcome struct {+ Status string+ StripeEventID string+ ErrorCode string+ ErrorMessage string+ SentAt time.Time+}++type Store interface {+ TryLock(ctx context.Context) (bool, func(context.Context) error, error)+ LatestLiveCutover(ctx context.Context) (time.Time, bool, error)+ CompletedHours(ctx context.Context, clusterID string, from, to time.Time) ([]time.Time, error)+ AggregateTenantHours(ctx context.Context, clusterID string, hours []time.Time) ([]TenantHour, error)+ LedgerHeads(ctx context.Context, hours []time.Time) (map[LedgerKey]LedgerHead, error)+ CreateBatch(ctx context.Context, batch Batch, submissions []Submission) ([]Submission, error)+ IncompleteBatches(ctx context.Context) ([]Batch, error)+ BatchSubmissions(ctx context.Context, batchID uuid.UUID) ([]Submission, error)+ MarkBatchArchived(ctx context.Context, batchID uuid.UUID, sha256 string, at time.Time) error+ MarkBatchCompleted(ctx context.Context, batchID uuid.UUID, at time.Time) error+ MarkSubmission(ctx context.Context, submissionID uuid.UUID, outcome Outcome) error+}++// Archiver writes one object. Implementations must not retry silently in a+// way that hides a permanent failure.+type Archiver interface {+ Put(ctx context.Context, bucket, key, contentType string, body []byte) error+}++// CustomerResolver maps a capsule tenant to a Stripe customer. The dry-run+// binary uses NoCustomers until the Stripe gateway is wired in.+type CustomerResolver interface {+ // Resolve returns the Stripe customer id, or found=false when the tenant+ // has no customer yet. A lookup failure is an error, never found=false.+ Resolve(ctx context.Context, tenant string) (customerID string, found bool, err error)+}++type NoCustomers struct{}++func (NoCustomers) Resolve(context.Context, string) (string, bool, error) { return "", false, nil }++type Result struct {+ BatchID uuid.UUID+ ResumedBatches int+ HoursConsidered int+ TenantHours int+ Created int+ Unchanged int+ Superseded int+ DryRun int+ Unmapped int+ OldestUnsentHour time.Time+}++type Runner struct {+ Config Config+ Store Store+ Archiver Archiver+ Customers CustomerResolver+ Now func() time.Time+ NewUUID func() uuid.UUID+}++func (r Runner) Run(ctx context.Context) (Result, error) {+ if err := r.Config.validate(); err != nil {+ return Result{}, err+ }+ if r.Store == nil || r.Archiver == nil {+ return Result{}, errors.New("store and archiver are required")+ }+ if r.Customers == nil {+ r.Customers = NoCustomers{}+ }+ if r.Now == nil {+ r.Now = time.Now+ }+ if r.NewUUID == nil {+ r.NewUUID = uuid.New+ }++ locked, unlock, err := r.Store.TryLock(ctx)+ if err != nil {+ return Result{}, fmt.Errorf("acquire submitter lock: %w", err)+ }+ if !locked {+ return Result{}, ErrAlreadyRunning+ }+ result, runErr := r.run(ctx)+ if unlockErr := unlock(ctx); unlockErr != nil {+ return result, errors.Join(runErr, fmt.Errorf("release submitter lock: %w", unlockErr))+ }+ return result, runErr+}++var ErrAlreadyRunning = errors.New("another submitter run holds the lock")++func (r Runner) run(ctx context.Context) (Result, error) {+ var result Result++ liveCutover, hasLive, err := r.Store.LatestLiveCutover(ctx)+ if err != nil {+ return result, fmt.Errorf("read live cutover: %w", err)+ }+ if hasLive && r.Config.CutoverHour.Before(liveCutover) {+ return result, fmt.Errorf("configured cutover %s is earlier than the live batch cutover %s", r.Config.CutoverHour.Format(time.RFC3339), liveCutover.Format(time.RFC3339))+ }++ incomplete, err := r.Store.IncompleteBatches(ctx)+ if err != nil {+ return result, fmt.Errorf("list incomplete batches: %w", err)+ }+ for _, batch := range incomplete {+ if err := r.completeBatch(ctx, batch, &result); err != nil {+ return result, fmt.Errorf("resume batch %s: %w", batch.BatchID, err)+ }+ result.ResumedBatches+++ }++ now := r.Now().UTC()+ windowEnd := now.Add(-r.Config.SettlementLag).Truncate(time.Hour)+ windowStart := now.Add(-r.Config.Lookback).Truncate(time.Hour)+ if windowStart.Before(r.Config.CutoverHour) {+ windowStart = r.Config.CutoverHour+ }+ if !windowEnd.After(windowStart) {+ return result, nil+ }++ hours, err := r.Store.CompletedHours(ctx, r.Config.ClusterID, windowStart, windowEnd)+ if err != nil {+ return result, fmt.Errorf("list completed hours: %w", err)+ }+ result.HoursConsidered = len(hours)+ if len(hours) == 0 {+ return result, nil+ }++ tenantHours, err := r.Store.AggregateTenantHours(ctx, r.Config.ClusterID, hours)+ if err != nil {+ return result, fmt.Errorf("aggregate tenant hours: %w", err)+ }+ result.TenantHours = len(tenantHours)+ heads, err := r.Store.LedgerHeads(ctx, hours)+ if err != nil {+ return result, fmt.Errorf("read ledger heads: %w", err)+ }++ batchID := r.NewUUID()+ var submissions []Submission+ var superseded []uuid.UUID+ customers := map[string]customerLookup{}+ for _, tenantHour := range tenantHours {+ for _, meter := range Meters {+ hash, err := FactSetSHA256(meter, tenantHour.Facts)+ if err != nil {+ return result, err+ }+ key := LedgerKey{Tenant: tenantHour.Tenant, HourStart: tenantHour.HourStart, Meter: meter}+ head, exists := heads[key]+ seq := 1+ if exists {+ if head.FactSetSHA256 == hash && (head.Status == StatusSent || head.Status == StatusDryRun || head.Status == StatusArchived) {+ result.Unchanged+++ if head.Status == StatusArchived {+ result.OldestUnsentHour = olderOf(result.OldestUnsentHour, tenantHour.HourStart)+ }+ continue+ }+ if head.Status == StatusArchived {+ superseded = append(superseded, head.SubmissionID)+ }+ seq = head.Seq + 1+ }+ identifier, err := Identifier(meter, tenantHour.Tenant, tenantHour.HourStart, seq)+ if err != nil {+ return result, err+ }+ quantity, err := tenantHour.Quantity(meter)+ if err != nil {+ return result, err+ }+ lookup, cached := customers[tenantHour.Tenant]+ if !cached {+ lookup.id, lookup.found, err = r.Customers.Resolve(ctx, tenantHour.Tenant)+ if err != nil {+ return result, fmt.Errorf("resolve customer for tenant %q: %w", tenantHour.Tenant, err)+ }+ customers[tenantHour.Tenant] = lookup+ }+ submissions = append(submissions, Submission{+ SubmissionID: r.NewUUID(),+ BatchID: batchID,+ Tenant: tenantHour.Tenant,+ HourStart: tenantHour.HourStart,+ Meter: meter,+ Seq: seq,+ Identifier: identifier,+ StripeCustomerID: lookup.id,+ Quantity: quantity,+ FactSetSHA256: hash,+ Status: StatusArchived,+ CreatedAt: now,+ })+ result.OldestUnsentHour = olderOf(result.OldestUnsentHour, tenantHour.HourStart)+ }+ }+ if len(submissions) == 0 {+ return result, nil+ }++ for _, submissionID := range superseded {+ if err := r.Store.MarkSubmission(ctx, submissionID, Outcome{Status: StatusFailed, ErrorCode: ErrorCodeSuperseded, ErrorMessage: "facts changed before completion; superseded by a newer seq"}); err != nil {+ return result, fmt.Errorf("supersede in-flight submission %s: %w", submissionID, err)+ }+ result.Superseded+++ }++ batch := Batch{+ BatchID: batchID,+ ClusterID: r.Config.ClusterID,+ CutoverHour: r.Config.CutoverHour,+ WindowStart: windowStart,+ WindowEnd: windowEnd,+ DryRun: r.Config.DryRun,+ ArchiveBucket: r.Config.ArchiveBucket,+ ArchiveKey: archiveDataKey(r.Config.ArchivePrefix, batchID),+ SubmissionCount: len(submissions),+ CreatedAt: now,+ }+ created, err := r.Store.CreateBatch(ctx, batch, submissions)+ if err != nil {+ return result, fmt.Errorf("create batch: %w", err)+ }+ result.BatchID = batchID+ result.Created = len(created)+ if err := r.completeBatch(ctx, batch, &result); err != nil {+ return result, fmt.Errorf("complete batch %s: %w", batchID, err)+ }+ return result, nil+}++type customerLookup struct {+ id string+ found bool+}++// completeBatch archives a batch (if not yet archived) and applies outcomes to+// every row still in the archived state. It is idempotent so a killed run can+// be resumed by the next one.+func (r Runner) completeBatch(ctx context.Context, batch Batch, result *Result) error {+ rows, err := r.Store.BatchSubmissions(ctx, batch.BatchID)+ if err != nil {+ return fmt.Errorf("read batch submissions: %w", err)+ }+ if batch.ArchivedAt.IsZero() {+ archive, err := BuildArchive(batch, rows, r.Config.ExporterVersion, r.Now().UTC())+ if err != nil {+ return fmt.Errorf("build archive: %w", err)+ }+ if err := r.Archiver.Put(ctx, batch.ArchiveBucket, batch.ArchiveKey, "application/x-ndjson", archive.Data); err != nil {+ return fmt.Errorf("write archive data: %w", err)+ }+ if err := r.Archiver.Put(ctx, batch.ArchiveBucket, archiveManifestKey(batch.ArchiveKey), "application/json", archive.Manifest); err != nil {+ return fmt.Errorf("write archive manifest: %w", err)+ }+ if err := r.Store.MarkBatchArchived(ctx, batch.BatchID, archive.SHA256, r.Now().UTC()); err != nil {+ return fmt.Errorf("mark batch archived: %w", err)+ }+ }+ if !batch.DryRun {+ return errors.New("live submission is not implemented")+ }+ for _, row := range rows {+ if row.Status != StatusArchived {+ continue+ }+ if err := r.Store.MarkSubmission(ctx, row.SubmissionID, Outcome{Status: StatusDryRun}); err != nil {+ return fmt.Errorf("mark submission %s: %w", row.Identifier, err)+ }+ result.DryRun+++ }+ if err := r.Store.MarkBatchCompleted(ctx, batch.BatchID, r.Now().UTC()); err != nil {+ return fmt.Errorf("mark batch completed: %w", err)+ }+ return nil+}++func olderOf(current, candidate time.Time) time.Time {+ if current.IsZero() || candidate.Before(current) {+ return candidate+ }+ return current+}++func archiveDataKey(prefix string, batchID uuid.UUID) string {+ prefix = strings.Trim(prefix, "/")+ key := fmt.Sprintf("schema=%s/%s/batch=%s/submissions.jsonl", SchemaVersion, ArchiveDataset, batchID)+ if prefix == "" {+ return key+ }+ return prefix + "/" + key+}++func archiveManifestKey(dataKey string) string {+ return strings.TrimSuffix(dataKey, "submissions.jsonl") + "manifest.json"+}diff --git a/libs/fleet/backend/submitter/submitter_test.go b/libs/fleet/backend/submitter/submitter_test.gonew file mode 100644index 0000000000..119a1d8ccf--- /dev/null+++ b/libs/fleet/backend/submitter/submitter_test.go@@ -0,0 +1,473 @@+package submitter++import (+ "context"+ "encoding/json"+ "errors"+ "strconv"+ "strings"+ "testing"+ "time"++ "github.com/google/uuid"+)++type fakeStore struct {+ locked bool+ liveCutover *time.Time+ hours []time.Time+ tenantHours []TenantHour+ heads map[LedgerKey]LedgerHead+ batches map[uuid.UUID]*Batch+ submissions map[uuid.UUID]*Submission+ failCreate error+ marks []Outcome+ createdOrder []uuid.UUID+}++func newFakeStore() *fakeStore {+ return &fakeStore{+ heads: map[LedgerKey]LedgerHead{},+ batches: map[uuid.UUID]*Batch{},+ submissions: map[uuid.UUID]*Submission{},+ }+}++func (s *fakeStore) TryLock(context.Context) (bool, func(context.Context) error, error) {+ if s.locked {+ return false, func(context.Context) error { return nil }, nil+ }+ s.locked = true+ return true, func(context.Context) error { s.locked = false; return nil }, nil+}++func (s *fakeStore) LatestLiveCutover(context.Context) (time.Time, bool, error) {+ if s.liveCutover == nil {+ return time.Time{}, false, nil+ }+ return *s.liveCutover, true, nil+}++func (s *fakeStore) CompletedHours(_ context.Context, _ string, from, to time.Time) ([]time.Time, error) {+ var hours []time.Time+ for _, hour := range s.hours {+ if !hour.Before(from) && hour.Before(to) {+ hours = append(hours, hour)+ }+ }+ return hours, nil+}++func (s *fakeStore) AggregateTenantHours(_ context.Context, _ string, hours []time.Time) ([]TenantHour, error) {+ var result []TenantHour+ for _, tenantHour := range s.tenantHours {+ for _, hour := range hours {+ if tenantHour.HourStart.Equal(hour) {+ result = append(result, tenantHour)+ }+ }+ }+ return result, nil+}++func (s *fakeStore) LedgerHeads(context.Context, []time.Time) (map[LedgerKey]LedgerHead, error) {+ heads := map[LedgerKey]LedgerHead{}+ for key, head := range s.heads {+ heads[key] = head+ }+ return heads, nil+}++func (s *fakeStore) CreateBatch(_ context.Context, batch Batch, submissions []Submission) ([]Submission, error) {+ if err := s.failCreate; err != nil {+ return nil, err+ }+ stored := batch+ s.batches[batch.BatchID] = &stored+ for _, submission := range submissions {+ row := submission+ s.submissions[submission.SubmissionID] = &row+ s.createdOrder = append(s.createdOrder, submission.SubmissionID)+ s.heads[LedgerKey{Tenant: row.Tenant, HourStart: row.HourStart, Meter: row.Meter}] = LedgerHead{+ SubmissionID: row.SubmissionID, BatchID: row.BatchID, Seq: row.Seq, Status: row.Status, FactSetSHA256: row.FactSetSHA256,+ }+ }+ return submissions, nil+}++func (s *fakeStore) IncompleteBatches(context.Context) ([]Batch, error) {+ var batches []Batch+ for _, batch := range s.batches {+ if batch.CompletedAt.IsZero() {+ batches = append(batches, *batch)+ }+ }+ return batches, nil+}++func (s *fakeStore) BatchSubmissions(_ context.Context, batchID uuid.UUID) ([]Submission, error) {+ var rows []Submission+ for _, id := range s.createdOrder {+ if row := s.submissions[id]; row.BatchID == batchID {+ rows = append(rows, *row)+ }+ }+ return rows, nil+}++func (s *fakeStore) MarkBatchArchived(_ context.Context, batchID uuid.UUID, sha string, at time.Time) error {+ batch, ok := s.batches[batchID]+ if !ok || !batch.ArchivedAt.IsZero() {+ return errors.New("batch not in unarchived state")+ }+ batch.ArchiveSHA256 = sha+ batch.ArchivedAt = at+ return nil+}++func (s *fakeStore) MarkBatchCompleted(_ context.Context, batchID uuid.UUID, at time.Time) error {+ batch, ok := s.batches[batchID]+ if !ok || !batch.CompletedAt.IsZero() {+ return errors.New("batch not in incomplete state")+ }+ batch.CompletedAt = at+ return nil+}++func (s *fakeStore) MarkSubmission(_ context.Context, submissionID uuid.UUID, outcome Outcome) error {+ row, ok := s.submissions[submissionID]+ if !ok || row.Status != StatusArchived {+ return errors.New("submission not in archived state")+ }+ row.Status = outcome.Status+ s.marks = append(s.marks, outcome)+ key := LedgerKey{Tenant: row.Tenant, HourStart: row.HourStart, Meter: row.Meter}+ head := s.heads[key]+ head.Status = outcome.Status+ s.heads[key] = head+ return nil+}++type fakeArchiver struct {+ objects map[string][]byte+ order []string+ fail error+}++func newFakeArchiver() *fakeArchiver { return &fakeArchiver{objects: map[string][]byte{}} }++func (a *fakeArchiver) Put(_ context.Context, bucket, key, _ string, body []byte) error {+ if err := a.fail; err != nil {+ return err+ }+ a.objects[bucket+"/"+key] = body+ a.order = append(a.order, key)+ return nil+}++var (+ cutover = time.Date(2026, 9, 15, 0, 0, 0, 0, time.UTC)+ h0 = time.Date(2026, 9, 15, 10, 0, 0, 0, time.UTC)+ h1 = h0.Add(time.Hour)+ h2 = h0.Add(2 * time.Hour)+ now = time.Date(2026, 9, 15, 15, 20, 0, 0, time.UTC)+)++// workedExample is the CUA-1150 example: 4 vCPU/8 GiB for 1h, 2 vCPU/4 GiB+// for 30 min, 8 vCPU/32 GiB for 2.5h, all starting at h0.+func workedExample() []TenantHour {+ gib := func(n int64) string { return itoa(n * 1073741824) }+ return []TenantHour{+ {Tenant: "user-acme", HourStart: h0, VCPUHours: "13.000000", GiBHours: "42.000000", Facts: []FactRef{+ {FactID: "a-h0", Revision: 1, CPUCoreSeconds: "14400.000000", MemoryByteSeconds: gib(8 * 3600)},+ {FactID: "b-h0", Revision: 1, CPUCoreSeconds: "3600.000000", MemoryByteSeconds: gib(4 * 1800)},+ {FactID: "c-h0", Revision: 1, CPUCoreSeconds: "28800.000000", MemoryByteSeconds: gib(32 * 3600)},+ }},+ {Tenant: "user-acme", HourStart: h1, VCPUHours: "8.000000", GiBHours: "32.000000", Facts: []FactRef{+ {FactID: "c-h1", Revision: 1, CPUCoreSeconds: "28800.000000", MemoryByteSeconds: gib(32 * 3600)},+ }},+ {Tenant: "user-acme", HourStart: h2, VCPUHours: "4.000000", GiBHours: "16.000000", Facts: []FactRef{+ {FactID: "c-h2", Revision: 1, CPUCoreSeconds: "14400.000000", MemoryByteSeconds: gib(32 * 1800)},+ }},+ }+}++func itoa(n int64) string { return strconv.FormatInt(n, 10) }++func testConfig() Config {+ return Config{+ ClusterID: "kopf-k3s",+ CutoverHour: cutover,+ SettlementLag: 2 * time.Hour,+ Lookback: 34 * 24 * time.Hour,+ DryRun: true,+ ArchiveBucket: "bucket",+ ArchivePrefix: "",+ ExporterVersion: "test",+ }+}++func newRunner(store *fakeStore, archiver *fakeArchiver) Runner {+ counter := 0+ return Runner{+ Config: testConfig(),+ Store: store,+ Archiver: archiver,+ Now: func() time.Time { return now },+ NewUUID: func() uuid.UUID {+ counter+++ return uuid.MustParse("00000000-0000-0000-0000-" + strings.Repeat("0", 12-len(itoa(int64(counter)))) + itoa(int64(counter)))+ },+ }+}++func TestRunCreatesOneSubmissionPerTenantHourMeterAndArchivesBeforeMarking(t *testing.T) {+ store := newFakeStore()+ store.hours = []time.Time{h0, h1, h2}+ store.tenantHours = workedExample()+ archiver := newFakeArchiver()++ result, err := newRunner(store, archiver).Run(context.Background())+ if err != nil {+ t.Fatalf("run: %v", err)+ }+ if result.Created != 6 || result.DryRun != 6 || result.Unchanged != 0 {+ t.Fatalf("result = %+v, want 6 created and 6 dry_run", result)+ }+ if len(archiver.order) != 2 || !strings.HasSuffix(archiver.order[0], "submissions.jsonl") || !strings.HasSuffix(archiver.order[1], "manifest.json") {+ t.Fatalf("archive order = %v, want data then manifest", archiver.order)+ }+ batch := store.batches[result.BatchID]+ if batch.ArchivedAt.IsZero() || batch.CompletedAt.IsZero() || batch.ArchiveSHA256 == "" {+ t.Fatalf("batch not archived and completed: %+v", *batch)+ }+ if batch.ArchiveKey != "schema=v1/reservation-usage/batch="+result.BatchID.String()+"/submissions.jsonl" {+ t.Fatalf("archive key = %q", batch.ArchiveKey)+ }+ var totals = map[string]string{}+ for _, row := range store.submissions {+ if row.Status != StatusDryRun {+ t.Fatalf("submission %s status = %s", row.Identifier, row.Status)+ }+ if row.Seq != 1 {+ t.Fatalf("submission %s seq = %d, want 1", row.Identifier, row.Seq)+ }+ totals[row.Meter+"@"+row.HourStart.Format(time.RFC3339)] = row.Quantity+ }+ if totals["cua_vcpu_hours@2026-09-15T10:00:00Z"] != "13.000000" || totals["cua_gib_hours@2026-09-15T10:00:00Z"] != "42.000000" {+ t.Fatalf("worked example H0 quantities = %v", totals)+ }+ data := archiver.objects["bucket/"+batch.ArchiveKey]+ if strings.Count(string(data), "\n") != 6 {+ t.Fatalf("archive rows = %d, want 6", strings.Count(string(data), "\n"))+ }+ var manifest archiveManifest+ if err := json.Unmarshal(archiver.objects["bucket/"+archiveManifestKey(batch.ArchiveKey)], &manifest); err != nil {+ t.Fatalf("decode manifest: %v", err)+ }+ if manifest.RowCount != 6 || manifest.DataSHA256 != batch.ArchiveSHA256 || !manifest.DryRun {+ t.Fatalf("manifest = %+v", manifest)+ }+}++func TestRunIsNoOpWhenFactsUnchanged(t *testing.T) {+ store := newFakeStore()+ store.hours = []time.Time{h0, h1, h2}+ store.tenantHours = workedExample()+ archiver := newFakeArchiver()+ runner := newRunner(store, archiver)+ if _, err := runner.Run(context.Background()); err != nil {+ t.Fatalf("first run: %v", err)+ }+ result, err := runner.Run(context.Background())+ if err != nil {+ t.Fatalf("second run: %v", err)+ }+ if result.Created != 0 || result.Unchanged != 6 || len(store.batches) != 1 {+ t.Fatalf("second run result = %+v, batches = %d", result, len(store.batches))+ }+}++func TestRunRevisionCreatesNextSeqForAffectedMeterOnly(t *testing.T) {+ store := newFakeStore()+ store.hours = []time.Time{h0}+ store.tenantHours = workedExample()[:1]+ archiver := newFakeArchiver()+ runner := newRunner(store, archiver)+ if _, err := runner.Run(context.Background()); err != nil {+ t.Fatalf("first run: %v", err)+ }+ // Revision 2 of fact c-h0 changes only the CPU seconds.+ store.tenantHours[0].Facts[2] = FactRef{FactID: "c-h0", Revision: 2, CPUCoreSeconds: "28000.000000", MemoryByteSeconds: store.tenantHours[0].Facts[2].MemoryByteSeconds}+ store.tenantHours[0].VCPUHours = "12.777777"++ result, err := runner.Run(context.Background())+ if err != nil {+ t.Fatalf("second run: %v", err)+ }+ if result.Created != 1 || result.Unchanged != 1 {+ t.Fatalf("result = %+v, want one new CPU submission and one unchanged memory submission", result)+ }+ var found bool+ for _, row := range store.submissions {+ if row.Meter == MeterVCPUHours && row.Seq == 2 {+ found = true+ if row.Identifier != "cua_vcpu_hours:user-acme:2026-09-15T10:00:00Z:r2" || row.Quantity != "12.777777" {+ t.Fatalf("seq 2 row = %+v", *row)+ }+ }+ }+ if !found {+ t.Fatal("expected a seq 2 CPU submission")+ }+}++func TestRunResumesBatchLeftUnarchivedByKilledRun(t *testing.T) {+ store := newFakeStore()+ store.hours = []time.Time{h0}+ store.tenantHours = workedExample()[:1]+ failing := newFakeArchiver()+ failing.fail = errors.New("s3 unavailable")+ runner := newRunner(store, failing)+ if _, err := runner.Run(context.Background()); err == nil || !strings.Contains(err.Error(), "s3 unavailable") {+ t.Fatalf("expected archive failure, got %v", err)+ }+ if len(store.batches) != 1 {+ t.Fatalf("batches = %d, want 1 in-flight", len(store.batches))+ }+ for _, row := range store.submissions {+ if row.Status != StatusArchived {+ t.Fatalf("row %s status = %s, want archived after failed archive", row.Identifier, row.Status)+ }+ }++ healthy := newFakeArchiver()+ runner.Archiver = healthy+ result, err := runner.Run(context.Background())+ if err != nil {+ t.Fatalf("resume run: %v", err)+ }+ if result.ResumedBatches != 1 || result.Created != 0 || result.DryRun != 2 || len(store.batches) != 1 {+ t.Fatalf("resume result = %+v, batches = %d", result, len(store.batches))+ }+ for _, row := range store.submissions {+ if row.Status != StatusDryRun || row.Seq != 1 {+ t.Fatalf("row %s after resume = status %s seq %d", row.Identifier, row.Status, row.Seq)+ }+ }+}++func TestRunRefusesCutoverEarlierThanLiveBatch(t *testing.T) {+ store := newFakeStore()+ live := cutover.Add(24 * time.Hour)+ store.liveCutover = &live+ _, err := newRunner(store, newFakeArchiver()).Run(context.Background())+ if err == nil || !strings.Contains(err.Error(), "earlier than the live batch cutover") {+ t.Fatalf("expected cutover refusal, got %v", err)+ }+}++func TestRunSkipsHoursBeforeCutoverAndInsideSettlementLag(t *testing.T) {+ store := newFakeStore()+ early := cutover.Add(-time.Hour)+ recent := now.Truncate(time.Hour).Add(-time.Hour) // hour_end is inside the 2h lag+ store.hours = []time.Time{early, h0, recent}+ store.tenantHours = append(workedExample()[:1],+ TenantHour{Tenant: "user-acme", HourStart: early, VCPUHours: "1.000000", GiBHours: "1.000000", Facts: []FactRef{{FactID: "e", Revision: 1, CPUCoreSeconds: "3600", MemoryByteSeconds: "3865470566400"}}},+ TenantHour{Tenant: "user-acme", HourStart: recent, VCPUHours: "1.000000", GiBHours: "1.000000", Facts: []FactRef{{FactID: "r", Revision: 1, CPUCoreSeconds: "3600", MemoryByteSeconds: "3865470566400"}}},+ )+ result, err := newRunner(store, newFakeArchiver()).Run(context.Background())+ if err != nil {+ t.Fatalf("run: %v", err)+ }+ if result.HoursConsidered != 1 || result.Created != 2 {+ t.Fatalf("result = %+v, want only h0 considered", result)+ }+}++func TestRunWhenLockHeldReturnsErrAlreadyRunning(t *testing.T) {+ store := newFakeStore()+ store.locked = true+ _, err := newRunner(store, newFakeArchiver()).Run(context.Background())+ if !errors.Is(err, ErrAlreadyRunning) {+ t.Fatalf("err = %v, want ErrAlreadyRunning", err)+ }+}++func TestConfigRejectsLiveMode(t *testing.T) {+ config := testConfig()+ config.DryRun = false+ if err := config.validate(); err == nil {+ t.Fatal("expected live mode to be rejected until implemented")+ }+}++func TestFactSetSHA256IsOrderIndependentAndMeterSpecific(t *testing.T) {+ facts := workedExample()[0].Facts+ reversed := []FactRef{facts[2], facts[1], facts[0]}+ a, err := FactSetSHA256(MeterVCPUHours, facts)+ if err != nil {+ t.Fatal(err)+ }+ b, err := FactSetSHA256(MeterVCPUHours, reversed)+ if err != nil {+ t.Fatal(err)+ }+ if a != b {+ t.Fatal("hash depends on fact order")+ }+ memory, err := FactSetSHA256(MeterGiBHours, facts)+ if err != nil {+ t.Fatal(err)+ }+ if memory == a {+ t.Fatal("cpu and memory hashes must differ")+ }+ if _, err := FactSetSHA256("cua_unknown", facts); err == nil {+ t.Fatal("unknown meter must be rejected")+ }+}++func TestIdentifierFormatAndLength(t *testing.T) {+ identifier, err := Identifier(MeterGiBHours, "user-acme", h0, 3)+ if err != nil {+ t.Fatal(err)+ }+ if identifier != "cua_gib_hours:user-acme:2026-09-15T10:00:00Z:r3" {+ t.Fatalf("identifier = %q", identifier)+ }+ if _, err := Identifier(MeterGiBHours, strings.Repeat("t", 80), h0, 1); err == nil {+ t.Fatal("expected identifier length rejection")+ }+ if _, err := Identifier(MeterGiBHours, "user-acme", h0, 0); err == nil {+ t.Fatal("expected seq validation")+ }+}++func TestBuildArchiveIsDeterministic(t *testing.T) {+ batchID := uuid.MustParse("11111111-1111-1111-1111-111111111111")+ batch := Batch{BatchID: batchID, ClusterID: "kopf-k3s", CutoverHour: cutover, WindowStart: h0, WindowEnd: h2, DryRun: true, ArchiveKey: archiveDataKey("", batchID)}+ rows := []Submission{+ {SubmissionID: uuid.MustParse("22222222-2222-2222-2222-222222222222"), BatchID: batchID, Tenant: "user-acme", HourStart: h0, Meter: MeterGiBHours, Seq: 1, Identifier: "cua_gib_hours:user-acme:2026-09-15T10:00:00Z:r1", Quantity: "42.000000", FactSetSHA256: strings.Repeat("b", 64)},+ {SubmissionID: uuid.MustParse("33333333-3333-3333-3333-333333333333"), BatchID: batchID, Tenant: "user-acme", HourStart: h0, Meter: MeterVCPUHours, Seq: 1, Identifier: "cua_vcpu_hours:user-acme:2026-09-15T10:00:00Z:r1", Quantity: "13.000000", FactSetSHA256: strings.Repeat("a", 64)},+ }+ first, err := BuildArchive(batch, rows, "test", now)+ if err != nil {+ t.Fatal(err)+ }+ second, err := BuildArchive(batch, []Submission{rows[1], rows[0]}, "test", now)+ if err != nil {+ t.Fatal(err)+ }+ if first.SHA256 != second.SHA256 || string(first.Data) != string(second.Data) {+ t.Fatal("archive depends on input order")+ }+ if !strings.HasPrefix(string(first.Data), `{"submission_id":"22222222`) {+ t.Fatalf("archive not sorted by identifier: %s", first.Data)+ }+ other := rows[0]+ other.BatchID = uuid.MustParse("44444444-4444-4444-4444-444444444444")+ if _, err := BuildArchive(batch, []Submission{other}, "test", now); err == nil {+ t.Fatal("expected foreign batch row rejection")+ }+}