Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(sync): refuse a schema dump read from an older state (#21847)
Danny Xu9771500469dc
diff --git a/backend/component/parsercontext/parsercontext_test.go b/backend/component/parsercontext/parsercontext_test.goindex f2af611dff..ceea0e8b8d 100644--- a/backend/component/parsercontext/parsercontext_test.go+++ b/backend/component/parsercontext/parsercontext_test.go@@ -3,6 +3,7 @@ package parsercontextimport ("context""testing"+ "time"metadatapb "github.com/bytebase/omni/metadata""github.com/stretchr/testify/require"@@ -53,11 +54,12 @@ func TestBuildGetLinkedDatabaseMetadataFunc(t *testing.T) {Metadata: &storepb.DatabaseMetadata{},})require.NoError(t, err)- require.NoError(t, s.UpsertDBSchema(ctx, instanceID, name, &metadatapb.DatabaseSchemaMetadata{+ _, err = s.UpsertDBSchema(ctx, instanceID, name, &metadatapb.DatabaseSchemaMetadata{Name: name,Schemas: []*metadatapb.SchemaMetadata{{Name: "", Tables: []*metadatapb.TableMetadata{{Name: "SECRET_T"}}}},LinkedDatabases: links,- }, &storepb.DatabaseConfig{}, nil))+ }, nil, time.Now())+ require.NoError(t, err)}}// The instance the statements run against; its data source reaches the loopback links.@@ -141,10 +143,11 @@ func TestBuildGetDatabaseMetadataFunc(t *testing.T) {_, err = s.UpsertDatabase(ctx, &store.DatabaseMessage{ProjectID: "project-a", InstanceID: "mssql", DatabaseName: name, Metadata: &storepb.DatabaseMetadata{}})require.NoError(t, err)}- require.NoError(t, s.UpsertDBSchema(ctx, "mssql", "synced", &metadatapb.DatabaseSchemaMetadata{+ _, err = s.UpsertDBSchema(ctx, "mssql", "synced", &metadatapb.DatabaseSchemaMetadata{Name: "synced",Schemas: []*metadatapb.SchemaMetadata{{Name: "dbo", Tables: []*metadatapb.TableMetadata{{Name: "t"}}}},- }, &storepb.DatabaseConfig{}, nil))+ }, nil, time.Now())+ require.NoError(t, err)get := BuildGetDatabaseMetadataFunc(s)diff --git a/backend/migrator/migration/3.25/0001##db_schema_synced_at.sql b/backend/migrator/migration/3.25/0001##db_schema_synced_at.sqlnew file mode 100644index 0000000000..97901a27e0--- /dev/null+++ b/backend/migrator/migration/3.25/0001##db_schema_synced_at.sql@@ -0,0 +1,16 @@+ALTER TABLE db_schema ADD COLUMN synced_at timestamptz;++-- Rows that predate the column have no record of when their dump was read.+-- db.metadata's lastSyncTime is the closest available value: when the database+-- last finished a sync, so later than the dump's read by that sync's duration,+-- and from a different sync than the one that wrote the dump where the two+-- raced. Being late is harmless only while the stamp stays in the past, which+-- LEAST enforces: lastSyncTime is whatever the clock said, and a stamp in the+-- future would refuse every sync until wall clock passed it. A database that+-- never recorded a sync keeps NULL, which every sync is allowed to overwrite.+UPDATE db_schema+SET synced_at = LEAST((db.metadata ->> 'lastSyncTime')::timestamptz, now())+FROM db+WHERE db.instance = db_schema.instance+ AND db.name = db_schema.db_name+ AND jsonb_typeof(db.metadata -> 'lastSyncTime') = 'string';diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sqlindex 493e806334..6ac2e8e85f 100644--- a/backend/migrator/migration/LATEST.sql+++ b/backend/migrator/migration/LATEST.sql@@ -587,6 +587,11 @@ CREATE TABLE db_schema (raw_dump text NOT NULL DEFAULT '',-- Stored as DatabaseConfig (proto/store/store/database.proto)config jsonb NOT NULL DEFAULT '{}',+ -- When the sync that produced metadata and raw_dump read the live database.+ -- A sync whose read predates this value is refused, so a slow sync cannot+ -- revert the projection to an older schema. NULL means never stamped, which+ -- every sync is allowed to overwrite.+ synced_at timestamptz,PRIMARY KEY (instance, db_name),CONSTRAINT db_schema_instance_db_name_fkey FOREIGN KEY(instance, db_name) REFERENCES db(instance, name));diff --git a/backend/migrator/migration_test.go b/backend/migrator/migration_test.goindex 05e13c0027..9fdbf714d7 100644--- a/backend/migrator/migration_test.go+++ b/backend/migrator/migration_test.go@@ -4,6 +4,7 @@ import ("context""database/sql""testing"+ "time""github.com/stretchr/testify/require"@@ -91,6 +92,77 @@ func TestMigration3_24_2_OracleTLSOff(t *testing.T) {}}+func TestMigration3_25_1_DBSchemaSyncedAtBackfill(t *testing.T) {+ ctx := context.Background()+ _, db := testcontainer.NewPgDatabase(t)+ _, err := db.ExecContext(ctx, `+ CREATE TABLE db (+ instance TEXT NOT NULL,+ name TEXT NOT NULL,+ metadata JSONB NOT NULL DEFAULT '{}',+ PRIMARY KEY (instance, name)+ );+ CREATE TABLE db_schema (+ instance TEXT NOT NULL,+ db_name TEXT NOT NULL,+ raw_dump TEXT NOT NULL DEFAULT '',+ PRIMARY KEY (instance, db_name)+ );+ INSERT INTO db (instance, name, metadata) VALUES+ -- protojson writes a Timestamp with nanosecond precision as nine+ -- fractional digits, more than timestamptz keeps.+ ('prod', 'synced', '{"lastSyncTime":"2026-10-28T04:05:06.123456789Z","syncStatus":"SYNC_STATUS_OK"}'),+ ('prod', 'whole-second', '{"lastSyncTime":"2026-10-28T04:05:06Z"}'),+ ('prod', 'never-synced', '{"syncStatus":"SYNC_STATUS_OK"}'),+ -- A clock that ran ahead, so the recorded time is in the future.+ ('prod', 'clock-ahead', '{"lastSyncTime":"2099-01-01T00:00:00Z"}'),+ -- Same database name in another instance: db_schema is keyed by+ -- (instance, db_name) and the backfill joins on the whole key.+ ('staging', 'synced', '{"lastSyncTime":"2026-10-20T01:02:03Z"}');+ INSERT INTO db_schema (instance, db_name, raw_dump) VALUES+ ('prod', 'synced', 'prod dump'),+ ('prod', 'whole-second', 'whole second dump'),+ ('prod', 'never-synced', 'unstamped dump'),+ ('prod', 'clock-ahead', 'clock ahead dump'),+ ('staging', 'synced', 'staging dump');+ `)+ require.NoError(t, err)++ statement, err := migrator.MigrationFS.ReadFile("migration/3.25/0001##db_schema_synced_at.sql")+ require.NoError(t, err)+ _, err = db.ExecContext(ctx, string(statement))+ require.NoError(t, err)++ for _, tc := range []struct {+ instance string+ databaseName string+ want string+ wantClamped bool+ }{+ {instance: "prod", databaseName: "synced", want: "2026-10-28T04:05:06.123457Z"},+ {instance: "prod", databaseName: "whole-second", want: "2026-10-28T04:05:06Z"},+ {instance: "prod", databaseName: "never-synced"},+ {instance: "prod", databaseName: "clock-ahead", wantClamped: true},+ {instance: "staging", databaseName: "synced", want: "2026-10-20T01:02:03Z"},+ } {+ var syncedAt sql.NullTime+ require.NoError(t, db.QueryRowContext(ctx,+ `SELECT synced_at FROM db_schema WHERE instance = $1 AND db_name = $2`,+ tc.instance, tc.databaseName).Scan(&syncedAt))+ switch {+ case tc.wantClamped:+ // A stamp in the future would refuse every sync until wall clock passed it.+ require.True(t, syncedAt.Valid, "%s/%s", tc.instance, tc.databaseName)+ require.WithinDuration(t, time.Now(), syncedAt.Time, time.Hour, "%s/%s", tc.instance, tc.databaseName)+ case tc.want == "":+ require.False(t, syncedAt.Valid, "%s/%s", tc.instance, tc.databaseName)+ default:+ require.True(t, syncedAt.Valid, "%s/%s", tc.instance, tc.databaseName)+ require.Equal(t, tc.want, syncedAt.Time.UTC().Format(time.RFC3339Nano), "%s/%s", tc.instance, tc.databaseName)+ }+ }+}+func TestMigration3_25_0_SQLExportPermission(t *testing.T) {ctx := context.Background()_, db := testcontainer.NewPgDatabase(t)diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex d2de4902ef..afbe145e1c 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -10,8 +10,8 @@ import (func TestLatestVersion(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)- require.Equal(t, semver.MustParse("3.25.0"), *files[len(files)-1].version)- require.Equal(t, "migration/3.25/0000##sql_export_permission.sql", files[len(files)-1].path)+ require.Equal(t, semver.MustParse("3.25.1"), *files[len(files)-1].version)+ require.Equal(t, "migration/3.25/0001##db_schema_synced_at.sql", files[len(files)-1].path)}func TestVersionUnique(t *testing.T) {diff --git a/backend/runner/reviewrun/ai_review_executor_test.go b/backend/runner/reviewrun/ai_review_executor_test.goindex 9d1cf521e9..3033040be6 100644--- a/backend/runner/reviewrun/ai_review_executor_test.go+++ b/backend/runner/reviewrun/ai_review_executor_test.go@@ -163,7 +163,8 @@ func setupReviewFixture(ctx context.Context, t *testing.T, issueCount int) (*sto})require.NoError(t, err)schema := &metadatapb.DatabaseSchemaMetadata{Name: name, Schemas: []*metadatapb.SchemaMetadata{{Name: "public"}}}- require.NoError(t, stores.UpsertDBSchema(ctx, "prod", name, schema, &storepb.DatabaseConfig{}, nil))+ _, err = stores.UpsertDBSchema(ctx, "prod", name, schema, nil, time.Now())+ require.NoError(t, err)}_, err = stores.CreateDatabaseGroup(ctx, &store.DatabaseGroupMessage{ProjectID: "p",diff --git a/backend/runner/schemasync/syncer.go b/backend/runner/schemasync/syncer.goindex 2e7374a0a2..3a812f278a 100644--- a/backend/runner/schemasync/syncer.go+++ b/backend/runner/schemasync/syncer.go@@ -28,7 +28,6 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/db""github.com/bytebase/bytebase/backend/store"- "github.com/bytebase/bytebase/backend/store/model")const (@@ -570,6 +569,9 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Datab// Sync database schemadeadlineCtx, cancelFunc := context.WithDeadline(ctx, time.Now().Add(syncTimeout))defer cancelFunc()+ // Everything below describes the database as of this instant, so it is what+ // orders this sync against the other syncs writing the same projection.+ readAt := time.Now()syncedDatabaseMetadata, err := driver.SyncDBSchema(deadlineCtx)if err != nil {return "", errors.Wrapf(err, "failed to sync database schema for database %q", database.DatabaseName)@@ -580,50 +582,15 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Datab}rawDump := schemaBuf.Bytes()- dbMetadata, err := s.store.GetDBSchema(ctx, &store.FindDBSchemaMessage{- Workspace: instance.Workspace,- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- })- if err != nil {- return "", errors.Wrapf(err, "failed to get database schema for database %q", database.DatabaseName)- }- // If the schema does not exist, then we create a new one.- // This happens when creating a new database in the test.- if dbMetadata == nil {- dbMetadata = model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{}, nil, &storepb.DatabaseConfig{}, instance.Metadata.GetEngine(), store.IsObjectCaseSensitive(instance))- }-- dbConfig := dbMetadata.GetConfig()-// Resolve store reads before UpdateDatabase opens its write transaction.// Acquiring another pool connection inside the callback can deadlock sync bursts.backupAvailable := s.databaseBackupAvailable(ctx, instance, syncedDatabaseMetadata)- // Build metadata updates- metadataUpdates := []func(*storepb.DatabaseMetadata){- func(md *storepb.DatabaseMetadata) {- md.LastSyncTime = timestamppb.Now()- md.BackupAvailable = backupAvailable- md.Datashare = syncedDatabaseMetadata.Datashare- md.SyncStatus = storepb.SyncStatus_SYNC_STATUS_OK- md.SyncError = ""- },- }-- if _, err := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- Deleted: new(false),- MetadataUpdates: metadataUpdates,- }); err != nil {- return "", errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)- }-- if err := s.store.UpsertDBSchema(ctx,+ stored, err := s.store.UpsertDBSchema(ctx,database.InstanceID, database.DatabaseName,- syncedDatabaseMetadata, dbConfig, rawDump,- ); err != nil {+ syncedDatabaseMetadata, rawDump, readAt,+ )+ if err != nil {if strings.Contains(err.Error(), "escape sequence") || strings.Contains(err.Error(), "invalid byte sequence") {if metadataBytes, err := protojson.Marshal(syncedDatabaseMetadata); err == nil {slog.Error("unsupported Unicode escape sequence", slog.String("metadata", string(metadataBytes)), slog.String("raw_dump", string(rawDump)))@@ -632,6 +599,34 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Databreturn "", errors.Wrapf(err, "failed to upsert database schema for database %q", database.DatabaseName)}+ // These values come from the same read, so leave them to the newer sync that won+ // rather than reporting this read as the database's current state. LastSyncTime+ // stays a completion time, because the sync interval is measured from it; the+ // dump's own freshness is db_schema.synced_at.+ if stored {+ if _, err := s.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ Deleted: new(false),+ MetadataUpdates: []func(*storepb.DatabaseMetadata){+ func(md *storepb.DatabaseMetadata) {+ md.LastSyncTime = timestamppb.Now()+ md.BackupAvailable = backupAvailable+ md.Datashare = syncedDatabaseMetadata.Datashare+ md.SyncStatus = storepb.SyncStatus_SYNC_STATUS_OK+ md.SyncError = ""+ },+ },+ }); err != nil {+ return "", errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)+ }+ } else {+ slog.Debug("Skipped stale database schema sync",+ slog.String("instance", database.InstanceID),+ slog.String("database", database.DatabaseName),+ slog.Time("readAt", readAt))+ }+// Create sync history if requestedif createSyncHistory {resourceID, err := s.store.CreateSyncHistory(ctx, database.InstanceID, database.DatabaseName, syncedDatabaseMetadata, string(rawDump))diff --git a/backend/runner/taskrun/database_migrate_executor.go b/backend/runner/taskrun/database_migrate_executor.goindex e8bbd09d70..ca27ce021b 100644--- a/backend/runner/taskrun/database_migrate_executor.go+++ b/backend/runner/taskrun/database_migrate_executor.go@@ -286,15 +286,7 @@ func (exec *DatabaseMigrateExecutor) runStandardMigration(ctx context.Context, dResourceID: changelogID,}if needDump {- opts.LogDatabaseSyncStart()- syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)- if err != nil {- opts.LogDatabaseSyncEnd(err.Error())- slog.ErrorContext(ctx, "failed to sync database schema", log.BBError(err))- } else {- opts.LogDatabaseSyncEnd("")- update.SyncHistory = &syncHistory- }+ update.SyncHistory = exec.dumpSchemaToHistory(ctx, database, &opts)}if migrationErr == nil {update.Status = new(store.ChangelogStatusDone)@@ -449,15 +441,7 @@ func (exec *DatabaseMigrateExecutor) runGhostMigration(ctx context.Context, drivupdate := &store.UpdateChangelogMessage{ResourceID: changelogID,}- opts.LogDatabaseSyncStart()- syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)- if err != nil {- opts.LogDatabaseSyncEnd(err.Error())- slog.ErrorContext(ctx, "failed to sync database schema", log.BBError(err))- } else {- opts.LogDatabaseSyncEnd("")- update.SyncHistory = &syncHistory- }+ update.SyncHistory = exec.dumpSchemaToHistory(ctx, database, &opts)if migrationErr == nil {update.Status = new(store.ChangelogStatusDone)} else {@@ -605,15 +589,7 @@ func (exec *DatabaseMigrateExecutor) runVersionedRelease(ctx context.Context, drupdate := &store.UpdateChangelogMessage{ResourceID: changelogID,}- opts.LogDatabaseSyncStart()- syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)- if err != nil {- opts.LogDatabaseSyncEnd(err.Error())- slog.ErrorContext(ctx, "failed to sync database schema", log.BBError(err))- } else {- opts.LogDatabaseSyncEnd("")- update.SyncHistory = &syncHistory- }+ update.SyncHistory = exec.dumpSchemaToHistory(ctx, database, &opts)if migrationErr == nil {update.Status = new(store.ChangelogStatusDone)} else {@@ -641,6 +617,25 @@ func (exec *DatabaseMigrateExecutor) runVersionedRelease(ctx context.Context, drreturn &storepb.TaskRunResult{}, nil}+// dumpSchemaToHistory records the post-migration schema as a sync history row for+// the changelog to point at, and returns nil when the dump failed. A failed dump+// does not fail the task: the migration itself already succeeded and there is+// nothing to roll back. It is reported by the task run log entry and by the+// changelog having no schema snapshot, and the database is queued for another+// sync so the stored schema stops lagging the migration it just applied.+func (exec *DatabaseMigrateExecutor) dumpSchemaToHistory(ctx context.Context, database *store.DatabaseMessage, opts *db.ExecuteOptions) *string {+ opts.LogDatabaseSyncStart()+ syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)+ if err != nil {+ opts.LogDatabaseSyncEnd(err.Error())+ slog.ErrorContext(ctx, "failed to sync database schema", log.BBError(err))+ exec.schemaSyncer.SyncDatabaseAsync(database)+ return nil+ }+ opts.LogDatabaseSyncEnd("")+ return &syncHistory+}+func (exec *DatabaseMigrateExecutor) runDeclarativeRelease(ctx context.Context, driverCtx context.Context, task *store.TaskMessage, taskRunUID int64, release *store.ReleaseMessage, instance *store.InstanceMessage, database *store.DatabaseMessage, project *store.ProjectMessage) (*storepb.TaskRunResult, error) {// Declarative releases should have exactly one fileif len(release.Payload.Files) == 0 {@@ -734,15 +729,7 @@ func (exec *DatabaseMigrateExecutor) runDeclarativeRelease(ctx context.Context,update := &store.UpdateChangelogMessage{ResourceID: changelogID,}- opts.LogDatabaseSyncStart()- syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)- if err != nil {- opts.LogDatabaseSyncEnd(err.Error())- slog.ErrorContext(ctx, "failed to sync database schema", log.BBError(err))- } else {- opts.LogDatabaseSyncEnd("")- update.SyncHistory = &syncHistory- }+ update.SyncHistory = exec.dumpSchemaToHistory(ctx, database, &opts)if migrationErr == nil {update.Status = new(store.ChangelogStatusDone)} else {diff --git a/backend/store/db_schema.go b/backend/store/db_schema.goindex 382da79728..4411a784ad 100644--- a/backend/store/db_schema.go+++ b/backend/store/db_schema.go@@ -3,6 +3,7 @@ package storeimport ("context""database/sql"+ "time"metadatapb "github.com/bytebase/omni/metadata""github.com/pkg/errors"@@ -75,22 +76,27 @@ func (s *Store) GetDBSchemaSnapshot(ctx context.Context, workspaceID string, insreturn s.GetDBSchema(ctx, &FindDBSchemaMessage{Workspace: workspaceID, InstanceID: instanceID, DatabaseName: databaseName})}-// UpsertDBSchema upserts a database schema.+// UpsertDBSchema upserts the schema a sync read from the live database at+// syncedAt. Syncs overlap - a migration task dumps the schema it just changed+// while the queued syncer is still dumping what it read before the migration -+// and the projection has no other ordering between them, so a write whose read+// predates the stored one is refused and reports stored=false. A refused write+// is an ordinary outcome: the row already holds a newer schema.+//+// config belongs to the catalog, not to the sync, so an existing row keeps its+// own. Writing it back from the value the sync read would revert a catalog+// update made while the sync was dumping.func (s *Store) UpsertDBSchema(ctx context.Context,instanceID,databaseName string,dbMetadata *metadatapb.DatabaseSchemaMetadata,- dbConfig *storepb.DatabaseConfig,rawDump []byte,-) error {+ syncedAt time.Time,+) (bool, error) {metadataBytes, err := protojson.Marshal(dbMetadata)if err != nil {- return err- }- configBytes, err := protojson.Marshal(dbConfig)- if err != nil {- return err+ return false, err}q := qb.Q().Space(`@@ -99,40 +105,49 @@ func (s *Store) UpsertDBSchema(db_name,metadata,raw_dump,- config+ synced_at)VALUES (?, ?, ?, ?, ?)ON CONFLICT(instance, db_name) DO UPDATE SETmetadata = EXCLUDED.metadata,raw_dump = EXCLUDED.raw_dump,- config = EXCLUDED.config- RETURNING metadata, raw_dump, config`,+ synced_at = EXCLUDED.synced_at+ WHERE db_schema.synced_at IS NULL OR db_schema.synced_at <= EXCLUDED.synced_at`,instanceID,databaseName,metadataBytes,// Convert to string because []byte{} is null which violates db schema constraints.string(rawDump),- configBytes)+ syncedAt)query, args, err := q.ToSQL()if err != nil {- return errors.Wrapf(err, "failed to build sql")+ return false, errors.Wrapf(err, "failed to build sql")}+ var stored boolerr = s.withDatabaseWrite(ctx, instanceID, databaseName, func(tx *sql.Tx) error {_, err := tx.ExecContext(ctx, "SELECT 1 FROM db_schema WHERE instance = $1 AND db_name = $2 FOR UPDATE", instanceID, databaseName)return err}, func(tx *sql.Tx, _ *databaseOwnership) error {- var metadata, schema, config []byte- return tx.QueryRowContext(ctx, query, args...).Scan(&metadata, &schema, &config)+ result, err := tx.ExecContext(ctx, query, args...)+ if err != nil {+ return err+ }+ rows, err := result.RowsAffected()+ if err != nil {+ return err+ }+ stored = rows > 0+ return nil})if err != nil {- return err+ return false, err}s.dbSchemaCache.Remove(getDBSchemaCacheKey(instanceID, databaseName))- return nil+ return stored, nil}// UpdateDBSchema updates a database schema.diff --git a/backend/store/db_schema_test.go b/backend/store/db_schema_test.gonew file mode 100644index 0000000000..dc3a8b6652--- /dev/null+++ b/backend/store/db_schema_test.go@@ -0,0 +1,246 @@+package store_test++import (+ "context"+ "database/sql"+ "testing"+ "time"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/store"+)++// TestUpsertDBSchemaRefusesOlderRead covers the lost update that made a migrated+// database read back its pre-migration schema: the queued syncer reads the live+// database before a migration runs and writes the projection after the migration+// task's own sync has already written it.+func TestUpsertDBSchemaRefusesOlderRead(t *testing.T) {+ t.Parallel()+ fixture := newStorePostgresFixture(t, `+ INSERT INTO instance (resource_id, workspace) VALUES ('instance-a', 'default'), ('instance-b', 'default');+ `)+ ctx, s := fixture.ctx, fixture.store++ // The same database name in two instances: db_schema is keyed by (instance, db_name).+ for _, instanceID := range []string{"instance-a", "instance-b"} {+ _, err := s.UpsertDatabase(ctx, &store.DatabaseMessage{+ ProjectID: "default",+ InstanceID: instanceID,+ DatabaseName: "shared",+ Metadata: &storepb.DatabaseMetadata{},+ })+ require.NoError(t, err)+ }++ readBefore := time.Now()+ readAfter := readBefore.Add(time.Minute)++ stored, err := s.UpsertDBSchema(ctx, "instance-a", "shared", schemaWithTable("before"), []byte("before"), readBefore)+ require.NoError(t, err)+ require.True(t, stored)++ stored, err = s.UpsertDBSchema(ctx, "instance-b", "shared", schemaWithTable("other"), []byte("other"), readAfter)+ require.NoError(t, err)+ require.True(t, stored)++ // The migration task's own sync.+ stored, err = s.UpsertDBSchema(ctx, "instance-a", "shared", schemaWithTable("after"), []byte("after"), readAfter)+ require.NoError(t, err)+ require.True(t, stored)+ requireStoredSchema(ctx, t, fixture, "instance-a", "shared", "after", readAfter)++ // The queued sync that read before the migration and writes after it.+ stored, err = s.UpsertDBSchema(ctx, "instance-a", "shared", schemaWithTable("before"), []byte("before"), readBefore)+ require.NoError(t, err)+ require.False(t, stored)+ requireStoredSchema(ctx, t, fixture, "instance-a", "shared", "after", readAfter)++ // A second read at the same instant is not older, so it still lands.+ stored, err = s.UpsertDBSchema(ctx, "instance-a", "shared", schemaWithTable("again"), []byte("again"), readAfter)+ require.NoError(t, err)+ require.True(t, stored)+ requireStoredSchema(ctx, t, fixture, "instance-a", "shared", "again", readAfter)++ // The refused write never reached the other instance's row.+ requireStoredSchema(ctx, t, fixture, "instance-b", "shared", "other", readAfter)+}++// TestUpsertDBSchemaStoresOverUnstampedRow covers the rows that predate the stamp:+// an upgraded deployment has synced_at NULL everywhere, so the first sync of each+// database wins however old its read is, and the guard orders every write after it.+// An unstamped row is why no backfill is needed - one write is enough to order the+// pair, whichever of the two lands on the NULL.+func TestUpsertDBSchemaStoresOverUnstampedRow(t *testing.T) {+ t.Parallel()+ fixture := newStorePostgresFixture(t, `+ INSERT INTO instance (resource_id, workspace) VALUES ('instance-a', 'default');+ `)+ ctx, s := fixture.ctx, fixture.store++ _, err := s.UpsertDatabase(ctx, &store.DatabaseMessage{+ ProjectID: "default",+ InstanceID: "instance-a",+ DatabaseName: "db",+ Metadata: &storepb.DatabaseMetadata{},+ })+ require.NoError(t, err)+ _, err = fixture.db.ExecContext(ctx,+ "INSERT INTO db_schema (instance, db_name, metadata, raw_dump) VALUES ($1, $2, $3, $4)",+ "instance-a", "db", `{"name":"db"}`, "before")+ require.NoError(t, err)++ // The older of two syncs racing across the upgrade reaches the NULL row first.+ olderRead := time.Now().Add(-time.Hour)+ stored, err := s.UpsertDBSchema(ctx, "instance-a", "db", schemaWithTable("older"), []byte("older"), olderRead)+ require.NoError(t, err)+ require.True(t, stored)+ requireStoredSchema(ctx, t, fixture, "instance-a", "db", "older", olderRead)++ // Its stamp is enough to order the newer sync behind it, and the next older read+ // behind that, so the window the NULL leaves open is one write wide.+ newerRead := olderRead.Add(time.Minute)+ stored, err = s.UpsertDBSchema(ctx, "instance-a", "db", schemaWithTable("newer"), []byte("newer"), newerRead)+ require.NoError(t, err)+ require.True(t, stored)+ requireStoredSchema(ctx, t, fixture, "instance-a", "db", "newer", newerRead)++ stored, err = s.UpsertDBSchema(ctx, "instance-a", "db", schemaWithTable("older"), []byte("older"), olderRead)+ require.NoError(t, err)+ require.False(t, stored)+ requireStoredSchema(ctx, t, fixture, "instance-a", "db", "newer", newerRead)+}++// TestUpsertDBSchemaRefusesOlderReadBlockedOnRowLock is the same ordering with the+// writes actually overlapping: the older read waits on the row lock the newer write+// holds, so it evaluates the guard against a row that changed under it.+func TestUpsertDBSchemaRefusesOlderReadBlockedOnRowLock(t *testing.T) {+ t.Parallel()+ fixture := newStorePostgresFixture(t, `+ INSERT INTO instance (resource_id, workspace) VALUES ('instance-a', 'default');+ `)+ ctx, s := fixture.ctx, fixture.store++ _, err := s.UpsertDatabase(ctx, &store.DatabaseMessage{+ ProjectID: "default",+ InstanceID: "instance-a",+ DatabaseName: "db",+ Metadata: &storepb.DatabaseMetadata{},+ })+ require.NoError(t, err)++ readBefore := time.Now()+ readAfter := readBefore.Add(time.Minute)++ stored, err := s.UpsertDBSchema(ctx, "instance-a", "db", schemaWithTable("before"), []byte("before"), readBefore)+ require.NoError(t, err)+ require.True(t, stored)++ // Stand in for the migration task's sync: hold the row lock UpsertDBSchema takes+ // first, write the post-migration schema, and only then let the older read in.+ tx, err := fixture.db.BeginTx(ctx, nil)+ require.NoError(t, err)+ defer tx.Rollback()+ _, err = tx.ExecContext(ctx, "SELECT 1 FROM db_schema WHERE instance = $1 AND db_name = $2 FOR UPDATE", "instance-a", "db")+ require.NoError(t, err)++ type result struct {+ stored bool+ err error+ }+ olderRead := make(chan result, 1)+ go func() {+ stored, err := s.UpsertDBSchema(ctx, "instance-a", "db", schemaWithTable("before"), []byte("before"), readBefore)+ olderRead <- result{stored: stored, err: err}+ }()++ // The older read must still be waiting on the lock; nothing else orders it.+ select {+ case r := <-olderRead:+ t.Fatalf("older read completed before the row lock was released: stored=%v err=%v", r.stored, r.err)+ case <-time.After(200 * time.Millisecond):+ }++ _, err = tx.ExecContext(ctx,+ "UPDATE db_schema SET metadata = $1, raw_dump = $2, synced_at = $3 WHERE instance = $4 AND db_name = $5",+ `{"name":"db","schemas":[{"name":"public","tables":[{"name":"after"}]}]}`, "after", readAfter, "instance-a", "db")+ require.NoError(t, err)+ require.NoError(t, tx.Commit())++ select {+ case r := <-olderRead:+ require.NoError(t, r.err)+ require.False(t, r.stored)+ case <-time.After(10 * time.Second):+ t.Fatal("older read never completed after the row lock was released")+ }+ requireStoredSchema(ctx, t, fixture, "instance-a", "db", "after", readAfter)+}++// TestUpsertDBSchemaKeepsCatalogConfig covers the other direction: a catalog edit+// made while a sync was dumping must survive that sync's write.+func TestUpsertDBSchemaKeepsCatalogConfig(t *testing.T) {+ t.Parallel()+ fixture := newStorePostgresFixture(t, `+ INSERT INTO instance (resource_id, workspace) VALUES ('instance-a', 'default');+ `)+ ctx, s := fixture.ctx, fixture.store++ _, err := s.UpsertDatabase(ctx, &store.DatabaseMessage{+ ProjectID: "default",+ InstanceID: "instance-a",+ DatabaseName: "db",+ Metadata: &storepb.DatabaseMetadata{},+ })+ require.NoError(t, err)++ readAt := time.Now()+ stored, err := s.UpsertDBSchema(ctx, "instance-a", "db", schemaWithTable("before"), []byte("before"), readAt)+ require.NoError(t, err)+ require.True(t, stored)++ config := &storepb.DatabaseConfig{+ Schemas: []*storepb.SchemaCatalog{{Name: "public", Tables: []*storepb.TableCatalog{{Name: "t", Classification: "1-1"}}}},+ }+ require.NoError(t, s.UpdateDBSchema(ctx, "instance-a", "db", &store.UpdateDBSchemaMessage{Config: config}))++ stored, err = s.UpsertDBSchema(ctx, "instance-a", "db", schemaWithTable("after"), []byte("after"), readAt.Add(time.Minute))+ require.NoError(t, err)+ require.True(t, stored)++ schema, err := s.GetDBSchema(ctx, &store.FindDBSchemaMessage{Workspace: "default", InstanceID: "instance-a", DatabaseName: "db"})+ require.NoError(t, err)+ require.Equal(t, "1-1", schema.GetConfig().GetSchemas()[0].GetTables()[0].GetClassification())+}++func schemaWithTable(table string) *metadatapb.DatabaseSchemaMetadata {+ return &metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{Name: "public", Tables: []*metadatapb.TableMetadata{{Name: table}}}},+ }+}++// requireStoredSchema asserts the projection a schema read serves, and the stamp+// that decides which read owns it, read past the schema cache.+func requireStoredSchema(ctx context.Context, t *testing.T, fixture *storePostgresFixture, instanceID, databaseName, table string, syncedAt time.Time) {+ t.Helper()+ var rawDump string+ var stored sql.NullTime+ require.NoError(t, fixture.db.QueryRowContext(ctx,+ "SELECT raw_dump, synced_at FROM db_schema WHERE instance = $1 AND db_name = $2",+ instanceID, databaseName).Scan(&rawDump, &stored))+ require.Equal(t, table, rawDump)+ require.True(t, stored.Valid)+ require.WithinDuration(t, syncedAt, stored.Time, time.Microsecond)++ schema, err := fixture.store.GetDBSchema(ctx, &store.FindDBSchemaMessage{+ Workspace: "default",+ InstanceID: instanceID,+ DatabaseName: databaseName,+ })+ require.NoError(t, err)+ require.NotNil(t, schema.GetSchemaMetadata("public").GetTable(table))+ require.Equal(t, table, string(schema.GetRawDump()))+}diff --git a/backend/tests/database.go b/backend/tests/database.goindex 13534f227c..1ae8293d84 100644--- a/backend/tests/database.go+++ b/backend/tests/database.go@@ -239,13 +239,9 @@ func (ctl *controller) waitDatabaseRegistered(ctx context.Context, instance *v1p// waitSchemaDump returns the database's schema dump once it reports want, a// string the change just applied to the database puts in the dump.//-// A completed rollout does not make that dump current. createDatabase registers-// the database, and the instance sync that registration runs queues an-// asynchronous schema sync for it; that queued sync can read the database-// before a later migration and write its dump after the migration's own sync-// wrote one, and db_schema keeps the last write, so the dump reverts to the-// pre-migration schema. Syncing once per round makes the dump current and-// outlasts such a write.+// A completed rollout does not make that dump current: the task's own+// post-migration sync is reported through the changelog, and a failed dump does+// not fail the task. Syncing once per round makes the dump current.func (ctl *controller) waitSchemaDump(ctx context.Context, databaseName, want string) (string, error) {ticker := time.NewTicker(pollInterval)defer ticker.Stop()