Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.
Rust agentai-agentapplecomputer-usecomputer-use-agentcontainerizationcuadesktop-automationhacktoberfestlumemacosmanusoperatorswiftvirtualizationvirtualization-frameworkwindowswindows-sandbox
fix(database): restore reservation migration checksum (#7155)
r33drichards3338a52df60c
diff --git a/libs/fleet/backend/cmd/usage-meter/main.go b/libs/fleet/backend/cmd/usage-meter/main.goindex df54aa9b52..6fcc06e208 100644--- a/libs/fleet/backend/cmd/usage-meter/main.go+++ b/libs/fleet/backend/cmd/usage-meter/main.go@@ -141,7 +141,10 @@ func loadConfig(now time.Time) (config, error) {cfg.hourStart = now.UTC().Truncate(time.Hour).Add(-time.Hour)if rawHour := os.Getenv("METER_HOUR_START"); rawHour != "" {cfg.hourStart, err = time.Parse(time.RFC3339, rawHour)- if err != nil || !cfg.hourStart.Equal(cfg.hourStart.UTC().Truncate(time.Hour)) {+ if err != nil {+ return config{}, fmt.Errorf("METER_HOUR_START must be an exact UTC RFC3339 hour: %w", err)+ }+ if !cfg.hourStart.Equal(cfg.hourStart.UTC().Truncate(time.Hour)) {return config{}, fmt.Errorf("METER_HOUR_START must be an exact UTC RFC3339 hour")}}@@ -154,7 +157,10 @@ func durationEnv(name string, fallback time.Duration) (time.Duration, error) {return fallback, nil}value, err := time.ParseDuration(raw)- if err != nil || value <= 0 {+ if err != nil {+ return 0, fmt.Errorf("%s must be a positive duration: %w", name, err)+ }+ if value <= 0 {return 0, fmt.Errorf("%s must be a positive duration", name)}return value, nildiff --git a/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql b/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sqlindex 1fa0971b57..eaf452032f 100644--- a/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql+++ b/libs/fleet/backend/database/migrations/000005_hourly_reservation_meter.sql@@ -3,9 +3,9 @@ CREATE ROLE cyclops_meter_writer LOGIN NOINHERIT NOCREATEROLE NOSUPERUSER NOCREAGRANT billing_meter_owner TO CURRENT_USER WITH INHERIT FALSE, SET TRUE;CREATE SCHEMA billing_meter AUTHORIZATION billing_meter_owner;-SET LOCAL ROLE billing_meter_owner;REVOKE CREATE ON SCHEMA billing_meter FROM PUBLIC;+SET LOCAL ROLE billing_meter_owner;CREATE TABLE billing_meter.reservation_hour_collection (collection_run_id uuid PRIMARY KEY,logical_key text NOT NULL,diff --git a/libs/fleet/backend/database/migrator.go b/libs/fleet/backend/database/migrator.goindex 284973eaba..6886d482d8 100644--- a/libs/fleet/backend/database/migrator.go+++ b/libs/fleet/backend/database/migrator.go@@ -398,6 +398,10 @@ func Run(ctx context.Context, config Config) (runErr error) {continue}+ file, err = prepareMigrationExecution(file)+ if err != nil {+ return fmt.Errorf("prepare migration %s: %w", file.Name, err)+ }if _, err := transaction.Exec(ctx, file.SQL); err != nil {return fmt.Errorf("apply migration %s: %w", file.Name, err)}diff --git a/libs/fleet/backend/database/migrator_test.go b/libs/fleet/backend/database/migrator_test.goindex fac1cf362d..ec7822dd98 100644--- a/libs/fleet/backend/database/migrator_test.go+++ b/libs/fleet/backend/database/migrator_test.go@@ -631,6 +631,48 @@ func TestEmbeddedMigrationsAreOrderedAndImmutable(t *testing.T) {}}+func TestReservationMeterMigrationRetainsRecordedChecksum(t *testing.T) {+ files, err := embeddedMigrations()+ if err != nil {+ t.Fatal(err)+ }+ meter := files[4]+ if meter.SHA256 != hourlyReservationMeterOriginalSHA256 {+ t.Fatalf("reservation meter checksum = %s, want recorded %s", meter.SHA256, hourlyReservationMeterOriginalSHA256)+ }+}++func TestExecutableMigrationSQLRepairsLegacyPrivilegeOrder(t *testing.T) {+ file := migrationFile{+ Version: 5,+ Name: "000005_hourly_reservation_meter.sql",+ SHA256: hourlyReservationMeterOriginalSHA256,+ SQL: hourlyReservationMeterPrivilegeSequence,+ }++ executable, err := prepareMigrationExecution(file)+ if err != nil {+ t.Fatal(err)+ }+ if executable.SQL != hourlyReservationMeterCompatiblePrivilegeSequence {+ t.Fatalf("executable migration SQL = %q", executable.SQL)+ }+ if file.SQL != hourlyReservationMeterPrivilegeSequence {+ t.Fatal("compatibility repair must not mutate immutable migration bytes")+ }+}++func TestExecutableMigrationSQLRejectsAmbiguousLegacyMigration(t *testing.T) {+ _, err := prepareMigrationExecution(migrationFile{+ Name: "000005_hourly_reservation_meter.sql",+ SHA256: hourlyReservationMeterOriginalSHA256,+ SQL: hourlyReservationMeterPrivilegeSequence + "\n" + hourlyReservationMeterPrivilegeSequence,+ })+ if err == nil || !strings.Contains(err.Error(), "missing or ambiguous") {+ t.Fatalf("expected ambiguous legacy sequence error, got %v", err)+ }+}+func TestAppliedMigrationLedgerUsesApplicationOrder(t *testing.T) {if !strings.Contains(createAppliedMigrationsTableStatement, "application_order bigint generated always as identity unique") {t.Fatalf("ledger schema must define a unique identity application order: %s", createAppliedMigrationsTableStatement)diff --git a/libs/fleet/backend/database/runtime_ddl.go b/libs/fleet/backend/database/runtime_ddl.goindex f7a5eb1e57..bb3cd9ec8e 100644--- a/libs/fleet/backend/database/runtime_ddl.go+++ b/libs/fleet/backend/database/runtime_ddl.go@@ -10,6 +10,17 @@ import ("github.com/jackc/pgx/v5")+const hourlyReservationMeterOriginalSHA256 = "fabda9a67ab9323b70a6d189a89be4194e1a76778a8c3afc77a0f1763f82bf4e"++const hourlyReservationMeterPrivilegeSequence = `CREATE SCHEMA billing_meter AUTHORIZATION billing_meter_owner;+REVOKE CREATE ON SCHEMA billing_meter FROM PUBLIC;++SET LOCAL ROLE billing_meter_owner;`++const hourlyReservationMeterCompatiblePrivilegeSequence = `CREATE SCHEMA billing_meter AUTHORIZATION billing_meter_owner;+SET LOCAL ROLE billing_meter_owner;+REVOKE CREATE ON SCHEMA billing_meter FROM PUBLIC;`+const createAppliedMigrationsTableStatement = `create table if not exists cyclops_migrations.applied_migrations (application_order bigint generated always as identity unique,version bigint primary key,@@ -264,6 +275,20 @@ func staticRoleAlterClauses(contract staticRoleContract, attributes staticRoleAtreturn strings.Join(clauses, " "), nil}+func prepareMigrationExecution(file migrationFile) (migrationFile, error) {+ if file.SHA256 != hourlyReservationMeterOriginalSHA256 {+ return file, nil+ }+ if file.Name != "000005_hourly_reservation_meter.sql" {+ return migrationFile{}, fmt.Errorf("legacy reservation meter checksum belongs to unexpected migration %s", file.Name)+ }+ if strings.Count(file.SQL, hourlyReservationMeterPrivilegeSequence) != 1 {+ return migrationFile{}, fmt.Errorf("legacy reservation meter privilege sequence is missing or ambiguous")+ }+ file.SQL = strings.Replace(file.SQL, hourlyReservationMeterPrivilegeSequence, hourlyReservationMeterCompatiblePrivilegeSequence, 1)+ return file, nil+}+func newRuntimeDDL(transaction pgx.Tx) runtimeDDL {return runtimeDDL{transaction: transaction}}diff --git a/libs/fleet/backend/metering/postgres.go b/libs/fleet/backend/metering/postgres.goindex 47c31a29ab..31735fe4a0 100644--- a/libs/fleet/backend/metering/postgres.go+++ b/libs/fleet/backend/metering/postgres.go@@ -2,7 +2,6 @@ package meteringimport ("context"- "errors""fmt""strings""time"@@ -12,6 +11,21 @@ import ("github.com/jackc/pgx/v5/pgxpool")+const lockReservationHourCompletionStatement = `select pg_advisory_xact_lock(hashtextextended($1, 0))`++const selectReservationHourCompletionStatement = `select+ coalesce(current.collection_run_id, '00000000-0000-0000-0000-000000000000'::uuid),+ coalesce(current.revision, 0),+ coalesce(current.source_sha256, '')+ from (values (1)) as seed(value)+ left join billing_meter.reservation_hour_collection_current as current on current.logical_key = $1`++const insertReservationHourCompletionStatement = `insert into billing_meter.reservation_hour_collection (+ collection_run_id, logical_key, revision, cluster_id, hour_start, hour_end,+ covered_seconds, discovered_sandboxes, inserted_facts, unchanged_facts,+ source_sha256, supersedes_collection_run_id+) values ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12)`+type PostgresStore struct {pool *pgxpool.Pool}@@ -68,26 +82,29 @@ func (s *PostgresStore) AppendFact(ctx context.Context, fact HourFact) (bool, ervar previousID uuid.UUIDvar previousRevision intvar previousSHA string- err = transaction.QueryRow(ctx, `select fact_id, revision, source_sha256- from billing_meter.reservation_hour_current where logical_key = $1`, fact.LogicalKey).+ queryErr := transaction.QueryRow(ctx, `select+ coalesce(current.fact_id, '00000000-0000-0000-0000-000000000000'::uuid),+ coalesce(current.revision, 0),+ coalesce(current.source_sha256, '')+ from (values (1)) as seed(value)+ left join billing_meter.reservation_hour_current as current on current.logical_key = $1`, fact.LogicalKey).Scan(&previousID, &previousRevision, &previousSHA)- if err == nil && previousSHA == fact.SourceSHA256 {+ if queryErr != nil {+ return false, fmt.Errorf("read current reservation fact: %w", queryErr)+ }+ if previousRevision > 0 && previousSHA == fact.SourceSHA256 {if err := transaction.Commit(ctx); err != nil {return false, fmt.Errorf("commit unchanged reservation fact: %w", err)}return false, nil}- if err != nil && !errors.Is(err, pgx.ErrNoRows) {- return false, fmt.Errorf("read current reservation fact: %w", err)- }- revision := 1+ revision := previousRevision + 1var supersedes any- if err == nil {- revision = previousRevision + 1+ if previousRevision > 0 {supersedes = previousID}- _, err = transaction.Exec(ctx, `insert into billing_meter.reservation_hour_fact (+ _, insertErr := transaction.Exec(ctx, `insert into billing_meter.reservation_hour_fact (fact_id, logical_key, revision, cluster_id, capsule_tenant, namespace,sandbox_uid, sandbox_name, pool_name, runtime, hour_start, hour_end,virtual_cpu_core_seconds, virtual_memory_byte_seconds, ready_seconds,@@ -99,8 +116,8 @@ func (s *PostgresStore) AppendFact(ctx context.Context, fact HourFact) (bool, erfact.VirtualCPUCoreSeconds, fact.VirtualMemoryByteSeconds, fact.ReadySeconds,fact.CoveredSeconds, fact.ScrapeIntervalSeconds, fact.SourceSHA256, fact.CollectionRunID,supersedes)- if err != nil {- return false, fmt.Errorf("insert reservation fact: %w", err)+ if insertErr != nil {+ return false, fmt.Errorf("insert reservation fact: %w", insertErr)}if err := transaction.Commit(ctx); err != nil {return false, fmt.Errorf("commit reservation fact: %w", err)@@ -117,41 +134,35 @@ func (s *PostgresStore) CompleteHour(ctx context.Context, completion HourCompletreturn false, fmt.Errorf("begin hour completion transaction: %w", err)}defer func() { _ = transaction.Rollback(ctx) }()- if _, err := transaction.Exec(ctx, `select pg_advisory_xact_lock(hashtextextended(, 0))`, completion.LogicalKey); err != nil {+ if _, err := transaction.Exec(ctx, lockReservationHourCompletionStatement, completion.LogicalKey); err != nil {return false, fmt.Errorf("lock reservation hour completion: %w", err)}var previousID uuid.UUIDvar previousRevision intvar previousSHA string- err = transaction.QueryRow(ctx, `select collection_run_id, revision, source_sha256- from billing_meter.reservation_hour_collection_current where logical_key = `, completion.LogicalKey).+ queryErr := transaction.QueryRow(ctx, selectReservationHourCompletionStatement, completion.LogicalKey).Scan(&previousID, &previousRevision, &previousSHA)- if err == nil && previousSHA == completion.SourceSHA256 {+ if queryErr != nil {+ return false, fmt.Errorf("read current hour completion: %w", queryErr)+ }+ if previousRevision > 0 && previousSHA == completion.SourceSHA256 {if err := transaction.Commit(ctx); err != nil {return false, fmt.Errorf("commit unchanged hour completion: %w", err)}return false, nil}- if err != nil && !errors.Is(err, pgx.ErrNoRows) {- return false, fmt.Errorf("read current hour completion: %w", err)- }- revision := 1+ revision := previousRevision + 1var supersedes any- if err == nil {- revision = previousRevision + 1+ if previousRevision > 0 {supersedes = previousID}- _, err = transaction.Exec(ctx, `insert into billing_meter.reservation_hour_collection (- collection_run_id, logical_key, revision, cluster_id, hour_start, hour_end,- covered_seconds, discovered_sandboxes, inserted_facts, unchanged_facts,- source_sha256, supersedes_collection_run_id- ) values (,,,,,,,,,,,)`,+ _, insertErr := transaction.Exec(ctx, insertReservationHourCompletionStatement,completion.CollectionRunID, completion.LogicalKey, revision, completion.ClusterID,completion.HourStart, completion.HourEnd, completion.CoveredSeconds,completion.DiscoveredSandboxes, completion.InsertedFacts, completion.UnchangedFacts,completion.SourceSHA256, supersedes)- if err != nil {- return false, fmt.Errorf("insert hour completion: %w", err)+ if insertErr != nil {+ return false, fmt.Errorf("insert hour completion: %w", insertErr)}if err := transaction.Commit(ctx); err != nil {return false, fmt.Errorf("commit hour completion: %w", err)diff --git a/libs/fleet/backend/metering/postgres_test.go b/libs/fleet/backend/metering/postgres_test.gonew file mode 100644index 0000000000..b3cd88d8cb--- /dev/null+++ b/libs/fleet/backend/metering/postgres_test.go@@ -0,0 +1,24 @@+package metering++import (+ "reflect"+ "regexp"+ "sort"+ "strings"+ "testing"+)++func TestReservationHourCompletionStatementsRetainParameters(t *testing.T) {+ if strings.Count(lockReservationHourCompletionStatement, "$1") != 1 {+ t.Fatalf("hour completion lock statement = %q", lockReservationHourCompletionStatement)+ }+ if strings.Count(selectReservationHourCompletionStatement, "$1") != 1 {+ t.Fatalf("hour completion select statement = %q", selectReservationHourCompletionStatement)+ }+ parameters := regexp.MustCompile(`\$([0-9]+)`).FindAllString(insertReservationHourCompletionStatement, -1)+ sort.Strings(parameters)+ want := []string{"$1", "$10", "$11", "$12", "$2", "$3", "$4", "$5", "$6", "$7", "$8", "$9"}+ if !reflect.DeepEqual(parameters, want) {+ t.Fatalf("hour completion insert parameters = %v, want %v", parameters, want)+ }+}diff --git a/libs/fleet/backend/metering/prometheus.go b/libs/fleet/backend/metering/prometheus.goindex 7dcdea5327..f7012c0e22 100644--- a/libs/fleet/backend/metering/prometheus.go+++ b/libs/fleet/backend/metering/prometheus.go@@ -26,7 +26,10 @@ type PrometheusClient struct {func NewPrometheusClient(rawURL string, client *http.Client) (*PrometheusClient, error) {baseURL, err := url.Parse(rawURL)- if err != nil || (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" {+ if err != nil {+ return nil, fmt.Errorf("invalid Prometheus URL: %w", err)+ }+ if (baseURL.Scheme != "http" && baseURL.Scheme != "https") || baseURL.Host == "" {return nil, fmt.Errorf("invalid Prometheus URL")}if client == nil {