Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

fix(database): restore reservation migration checksum (#7155)

r33drichards3338a52df60c
diff --git a/libs/fleet/backend/cmd/usage-meter/main.go b/libs/fleet/backend/cmd/usage-meter/main.go
index df54aa9b52..6fcc06e208 100644
--- a/libs/fleet/backend/cmd/usage-meter/main.go
+++ b/libs/fleet/backend/cmd/usage-meter/main.go
@@ -141,7 +141,10 @@ func loadConfig(now time.Time) (config, error) {
cfg.hourStart = now.UTC().Truncate(time.Hour).Add(-time.Hour)
if rawHour := os.Getenv("METER_HOUR_START"); rawHour != "" {
cfg.hourStart, err = time.Parse(time.RFC3339, rawHour)
- if err != nil || !cfg.hourStart.Equal(cfg.hourStart.UTC().Truncate(time.Hour)) {
+ if err != nil {
+ return config{}, fmt.Errorf("METER_HOUR_START must be an exact UTC RFC3339 hour: %w", err)
+ }
+ if !cfg.hourStart.Equal(cfg.hourStart.UTC().Truncate(time.Hour)) {
return config{}, fmt.Errorf("METER_HOUR_START must be an exact UTC RFC3339 hour")
}
}
@@ -154,7 +157,10 @@ func durationEnv(name string, fallback time.Duration) (time.Duration, error) {
return fallback, nil
}
value, err := time.ParseDuration(raw)
- if err != nil || value <= 0 {
+ if err != nil {
+ return 0, fmt.Errorf("%s must be a positive duration: %w", name, err)
+ }
+ if value <= 0 {
return 0, fmt.Errorf("%s must be a positive duration", name)
}
return value, nil
diff --git a/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql b/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql
index 1fa0971b57..eaf452032f 100644
--- a/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql
+++ b/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql
@@ -3,9 +3,9 @@ CREATE ROLE cyclops_meter_writer LOGIN NOINHERIT NOCREATEROLE NOSUPERUSER NOCREA
GRANT billing_meter_owner TO CURRENT_USER WITH INHERIT FALSE, SET TRUE;
CREATE SCHEMA billing_meter AUTHORIZATION billing_meter_owner;
-SET LOCAL ROLE billing_meter_owner;
REVOKE CREATE ON SCHEMA billing_meter FROM PUBLIC;
+SET LOCAL ROLE billing_meter_owner;
CREATE TABLE billing_meter.reservation_hour_collection (
collection_run_id uuid PRIMARY KEY,
logical_key text NOT NULL,
diff --git a/libs/fleet/backend/database/migrator.go b/libs/fleet/backend/database/migrator.go
index 284973eaba..6886d482d8 100644
--- a/libs/fleet/backend/database/migrator.go
+++ b/libs/fleet/backend/database/migrator.go
@@ -398,6 +398,10 @@ func Run(ctx context.Context, config Config) (runErr error) {
continue
}
+ file, err = prepareMigrationExecution(file)
+ if err != nil {
+ return fmt.Errorf("prepare migration %s: %w", file.Name, err)
+ }
if _, err := transaction.Exec(ctx, file.SQL); err != nil {
return fmt.Errorf("apply migration %s: %w", file.Name, err)
}
diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.go
index fac1cf362d..ec7822dd98 100644
--- a/libs/fleet/backend/database/migrator_test.go
+++ b/libs/fleet/backend/database/migrator_test.go
@@ -631,6 +631,48 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {
}
}
+func TestReservationMeterMigrationRetainsRecordedChecksum(t *testing.T) {
+ files, err := embeddedMigrations()
+ if err != nil {
+ t.Fatal(err)
+ }
+ meter := files[4]
+ if meter.SHA256 != hourlyReservationMeterOriginalSHA256 {
+ t.Fatalf("reservation meter checksum = %s, want recorded %s", meter.SHA256, hourlyReservationMeterOriginalSHA256)
+ }
+}
+
+func TestExecutableMigrationSQLRepairsLegacyPrivilegeOrder(t *testing.T) {
+ file := migrationFile{
+ Version: 5,
+ Name: "000005_hourly_reservation_meter.sql",
+ SHA256: hourlyReservationMeterOriginalSHA256,
+ SQL: hourlyReservationMeterPrivilegeSequence,
+ }
+
+ executable, err := prepareMigrationExecution(file)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if executable.SQL != hourlyReservationMeterCompatiblePrivilegeSequence {
+ t.Fatalf("executable migration SQL = %q", executable.SQL)
+ }
+ if file.SQL != hourlyReservationMeterPrivilegeSequence {
+ t.Fatal("compatibility repair must not mutate immutable migration bytes")
+ }
+}
+
+func TestExecutableMigrationSQLRejectsAmbiguousLegacyMigration(t *testing.T) {
+ _, err := prepareMigrationExecution(migrationFile{
+ Name: "000005_hourly_reservation_meter.sql",
+ SHA256: hourlyReservationMeterOriginalSHA256,
+ SQL: hourlyReservationMeterPrivilegeSequence + "\n" + hourlyReservationMeterPrivilegeSequence,
+ })
+ if err == nil || !strings.Contains(err.Error(), "missing or ambiguous") {
+ t.Fatalf("expected ambiguous legacy sequence error, got %v", err)
+ }
+}
+
func TestAppliedMigrationLedgerUsesApplicationOrder(t *testing.T) {
if !strings.Contains(createAppliedMigrationsTableStatement, "application_order bigint generated always as identity unique") {
t.Fatalf("ledger schema must define a unique identity application order: %s", createAppliedMigrationsTableStatement)
diff --git a/libs/fleet/backend/database/runtime_ddl.go b/libs/fleet/backend/database/runtime_ddl.go
index f7a5eb1e57..bb3cd9ec8e 100644
--- a/libs/fleet/backend/database/runtime_ddl.go
+++ b/libs/fleet/backend/database/runtime_ddl.go
@@ -10,6 +10,17 @@ import (
"github.com/jackc/pgx/v5"
)
+const hourlyReservationMeterOriginalSHA256 = "fabda9a67ab9323b70a6d189a89be4194e1a76778a8c3afc77a0f1763f82bf4e"
+
+const hourlyReservationMeterPrivilegeSequence = `CREATE SCHEMA billing_meter AUTHORIZATION billing_meter_owner;
+REVOKE CREATE ON SCHEMA billing_meter FROM PUBLIC;
+
+SET LOCAL ROLE billing_meter_owner;`
+
+const hourlyReservationMeterCompatiblePrivilegeSequence = `CREATE SCHEMA billing_meter AUTHORIZATION billing_meter_owner;
+SET LOCAL ROLE billing_meter_owner;
+REVOKE CREATE ON SCHEMA billing_meter FROM PUBLIC;`
+
const createAppliedMigrationsTableStatement = `create table if not exists cyclops_migrations.applied_migrations (
application_order bigint generated always as identity unique,
version bigint primary key,
@@ -264,6 +275,20 @@ func staticRoleAlterClauses(contract staticRoleContract, attributes staticRoleAt
return strings.Join(clauses, " "), nil
}
+func prepareMigrationExecution(file migrationFile) (migrationFile, error) {
+ if file.SHA256 != hourlyReservationMeterOriginalSHA256 {
+ return file, nil
+ }
+ if file.Name != "000005_hourly_reservation_meter.sql" {
+ return migrationFile{}, fmt.Errorf("legacy reservation meter checksum belongs to unexpected migration %s", file.Name)
+ }
+ if strings.Count(file.SQL, hourlyReservationMeterPrivilegeSequence) != 1 {
+ return migrationFile{}, fmt.Errorf("legacy reservation meter privilege sequence is missing or ambiguous")
+ }
+ file.SQL = strings.Replace(file.SQL, hourlyReservationMeterPrivilegeSequence, hourlyReservationMeterCompatiblePrivilegeSequence, 1)
+ return file, nil
+}
+
func newRuntimeDDL(transaction pgx.Tx) runtimeDDL {
return runtimeDDL{transaction: transaction}
}
diff --git a/libs/fleet/backend/metering/postgres.go b/libs/fleet/backend/metering/postgres.go
index 47c31a29ab..31735fe4a0 100644
--- a/libs/fleet/backend/metering/postgres.go
+++ b/libs/fleet/backend/metering/postgres.go
@@ -2,7 +2,6 @@ package metering
import (
"context"
- "errors"
"fmt"
"strings"
"time"
@@ -12,6 +11,21 @@ import (
"github.com/jackc/pgx/v5/pgxpool"
)
+const lockReservationHourCompletionStatement = `select pg_advisory_xact_lock(hashtextextended($1, 0))`
+
+const selectReservationHourCompletionStatement = `select
+ coalesce(current.collection_run_id, '00000000-0000-0000-0000-000000000000'::uuid),
+ coalesce(current.revision, 0),
+ coalesce(current.source_sha256, '')
+ from (values (1)) as seed(value)
+ left join billing_meter.reservation_hour_collection_current as current on current.logical_key = $1`
+
+const insertReservationHourCompletionStatement = `insert into billing_meter.reservation_hour_collection (
+ collection_run_id, logical_key, revision, cluster_id, hour_start, hour_end,
+ covered_seconds, discovered_sandboxes, inserted_facts, unchanged_facts,
+ source_sha256, supersedes_collection_run_id
+) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`
+
type PostgresStore struct {
pool *pgxpool.Pool
}
@@ -68,26 +82,29 @@ func (s *PostgresStore) AppendFact(ctx context.Context, fact HourFact) (bool, er
var previousID uuid.UUID
var previousRevision int
var previousSHA string
- err = transaction.QueryRow(ctx, `select fact_id, revision, source_sha256
- from billing_meter.reservation_hour_current where logical_key = $1`, fact.LogicalKey).
+ queryErr := transaction.QueryRow(ctx, `select
+ coalesce(current.fact_id, '00000000-0000-0000-0000-000000000000'::uuid),
+ coalesce(current.revision, 0),
+ coalesce(current.source_sha256, '')
+ from (values (1)) as seed(value)
+ left join billing_meter.reservation_hour_current as current on current.logical_key = $1`, fact.LogicalKey).
Scan(&previousID, &previousRevision, &previousSHA)
- if err == nil && previousSHA == fact.SourceSHA256 {
+ if queryErr != nil {
+ return false, fmt.Errorf("read current reservation fact: %w", queryErr)
+ }
+ if previousRevision > 0 && previousSHA == fact.SourceSHA256 {
if err := transaction.Commit(ctx); err != nil {
return false, fmt.Errorf("commit unchanged reservation fact: %w", err)
}
return false, nil
}
- if err != nil && !errors.Is(err, pgx.ErrNoRows) {
- return false, fmt.Errorf("read current reservation fact: %w", err)
- }
- revision := 1
+ revision := previousRevision + 1
var supersedes any
- if err == nil {
- revision = previousRevision + 1
+ if previousRevision > 0 {
supersedes = previousID
}
- _, err = transaction.Exec(ctx, `insert into billing_meter.reservation_hour_fact (
+ _, insertErr := transaction.Exec(ctx, `insert into billing_meter.reservation_hour_fact (
fact_id, logical_key, revision, cluster_id, capsule_tenant, namespace,
sandbox_uid, sandbox_name, pool_name, runtime, hour_start, hour_end,
virtual_cpu_core_seconds, virtual_memory_byte_seconds, ready_seconds,
@@ -99,8 +116,8 @@ func (s *PostgresStore) AppendFact(ctx context.Context, fact HourFact) (bool, er
fact.VirtualCPUCoreSeconds, fact.VirtualMemoryByteSeconds, fact.ReadySeconds,
fact.CoveredSeconds, fact.ScrapeIntervalSeconds, fact.SourceSHA256, fact.CollectionRunID,
supersedes)
- if err != nil {
- return false, fmt.Errorf("insert reservation fact: %w", err)
+ if insertErr != nil {
+ return false, fmt.Errorf("insert reservation fact: %w", insertErr)
}
if err := transaction.Commit(ctx); err != nil {
return false, fmt.Errorf("commit reservation fact: %w", err)
@@ -117,41 +134,35 @@ func (s *PostgresStore) CompleteHour(ctx context.Context, completion HourComplet
return false, fmt.Errorf("begin hour completion transaction: %w", err)
}
defer func() { _ = transaction.Rollback(ctx) }()
- if _, err := transaction.Exec(ctx, `select pg_advisory_xact_lock(hashtextextended(, 0))`, completion.LogicalKey); err != nil {
+ if _, err := transaction.Exec(ctx, lockReservationHourCompletionStatement, completion.LogicalKey); err != nil {
return false, fmt.Errorf("lock reservation hour completion: %w", err)
}
var previousID uuid.UUID
var previousRevision int
var previousSHA string
- err = transaction.QueryRow(ctx, `select collection_run_id, revision, source_sha256
- from billing_meter.reservation_hour_collection_current where logical_key = `, completion.LogicalKey).
+ queryErr := transaction.QueryRow(ctx, selectReservationHourCompletionStatement, completion.LogicalKey).
Scan(&previousID, &previousRevision, &previousSHA)
- if err == nil && previousSHA == completion.SourceSHA256 {
+ if queryErr != nil {
+ return false, fmt.Errorf("read current hour completion: %w", queryErr)
+ }
+ if previousRevision > 0 && previousSHA == completion.SourceSHA256 {
if err := transaction.Commit(ctx); err != nil {
return false, fmt.Errorf("commit unchanged hour completion: %w", err)
}
return false, nil
}
- if err != nil && !errors.Is(err, pgx.ErrNoRows) {
- return false, fmt.Errorf("read current hour completion: %w", err)
- }
- revision := 1
+ revision := previousRevision + 1
var supersedes any
- if err == nil {
- revision = previousRevision + 1
+ if previousRevision > 0 {
supersedes = previousID
}
- _, err = transaction.Exec(ctx, `insert into billing_meter.reservation_hour_collection (
- collection_run_id, logical_key, revision, cluster_id, hour_start, hour_end,
- covered_seconds, discovered_sandboxes, inserted_facts, unchanged_facts,
- source_sha256, supersedes_collection_run_id
- ) values (,,,,,,,,,,,)`,
+ _, insertErr := transaction.Exec(ctx, insertReservationHourCompletionStatement,
completion.CollectionRunID, completion.LogicalKey, revision, completion.ClusterID,
completion.HourStart, completion.HourEnd, completion.CoveredSeconds,
completion.DiscoveredSandboxes, completion.InsertedFacts, completion.UnchangedFacts,
completion.SourceSHA256, supersedes)
- if err != nil {
- return false, fmt.Errorf("insert hour completion: %w", err)
+ if insertErr != nil {
+ return false, fmt.Errorf("insert hour completion: %w", insertErr)
}
if err := transaction.Commit(ctx); err != nil {
return false, fmt.Errorf("commit hour completion: %w", err)
diff --git a/libs/fleet/backend/metering/postgres_test.go b/libs/fleet/backend/metering/postgres_test.go
new file mode 100644
index 0000000000..b3cd88d8cb
--- /dev/null
+++ b/libs/fleet/backend/metering/postgres_test.go
@@ -0,0 +1,24 @@
+package metering
+
+import (
+ "reflect"
+ "regexp"
+ "sort"
+ "strings"
+ "testing"
+)
+
+func TestReservationHourCompletionStatementsRetainParameters(t *testing.T) {
+ if strings.Count(lockReservationHourCompletionStatement, "$1") != 1 {
+ t.Fatalf("hour completion lock statement = %q", lockReservationHourCompletionStatement)
+ }
+ if strings.Count(selectReservationHourCompletionStatement, "$1") != 1 {
+ t.Fatalf("hour completion select statement = %q", selectReservationHourCompletionStatement)
+ }
+ parameters := regexp.MustCompile(`\$([0-9]+)`).FindAllString(insertReservationHourCompletionStatement, -1)
+ sort.Strings(parameters)
+ want := []string{"$1", "$10", "$11", "$12", "$2", "$3", "$4", "$5", "$6", "$7", "$8", "$9"}
+ if !reflect.DeepEqual(parameters, want) {
+ t.Fatalf("hour completion insert parameters = %v, want %v", parameters, want)
+ }
+}
diff --git a/libs/fleet/backend/metering/prometheus.go b/libs/fleet/backend/metering/prometheus.go
index 7dcdea5327..f7012c0e22 100644
--- a/libs/fleet/backend/metering/prometheus.go
+++ b/libs/fleet/backend/metering/prometheus.go
@@ -26,7 +26,10 @@ type PrometheusClient struct {
func NewPrometheusClient(rawURL string, client *http.Client) (*PrometheusClient, error) {
baseURL, err := url.Parse(rawURL)
- if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" {
+ if err != nil {
+ return nil, fmt.Errorf("invalid Prometheus URL: %w", err)
+ }
+ if (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" {
return nil, fmt.Errorf("invalid Prometheus URL")
}
if client == nil {