Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(runner): simplify changelog management in database migrations (#18783)
Danny Xud17bdf31efac
diff --git a/backend/runner/taskrun/database_migrate_executor.go b/backend/runner/taskrun/database_migrate_executor.goindex 393ab2bff7..e984b17a61 100644--- a/backend/runner/taskrun/database_migrate_executor.go+++ b/backend/runner/taskrun/database_migrate_executor.go@@ -80,7 +80,18 @@ func (exec *DatabaseMigrateExecutor) RunOnce(ctx context.Context, driverCtx contreturn nil, errors.Wrap(err, "failed to ensure baseline changelog")}- // Check if this is a release-based task+ // Mark database as not drifted since we're about to sync it+ if _, err := exec.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ MetadataUpdates: []func(*storepb.DatabaseMetadata){func(md *storepb.DatabaseMetadata) {+ md.Drifted = false+ }},+ }); err != nil {+ return nil, errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)+ }++ // Execute migration based on task typeif releaseName := task.Payload.GetRelease(); releaseName != "" {// Parse release name to get project ID and release UID_, releaseUID, err := common.GetProjectReleaseUID(releaseName)@@ -201,7 +212,6 @@ func (exec *DatabaseMigrateExecutor) runStandardMigration(ctx context.Context, d}needDump := computeNeedDump(task.Type, database.Engine, sheet.Statement)- taskRunName := common.FormatTaskRun(database.ProjectID, task.PlanID, task.Environment, task.ID, taskRunUID)// Get database driverdriver, err := exec.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{@@ -229,47 +239,56 @@ func (exec *DatabaseMigrateExecutor) runStandardMigration(ctx context.Context, dreturn exec.store.CreateTaskRunLog(ctx, taskRunUID, t.UTC(), exec.profile.DeployID, e)}- // Begin migration - dump before migration- changelogUID, err := beginMigration(- ctx, exec.store, database,- taskRunName,- storepb.ChangelogPayload_MIGRATE, schema.GetDumpFormatVersion(instance.Metadata.GetEngine()), exec.profile.GitCommit,- )+ // Begin migration - create pending changelog+ changelogUID, err := exec.store.CreateChangelog(ctx, &store.ChangelogMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ Status: store.ChangelogStatusPending,+ SyncHistoryUID: nil,+ Payload: &storepb.ChangelogPayload{+ TaskRun: common.FormatTaskRun(database.ProjectID, task.PlanID, task.Environment, task.ID, taskRunUID),+ Type: storepb.ChangelogPayload_MIGRATE,+ GitCommit: exec.profile.GitCommit,+ DumpVersion: schema.GetDumpFormatVersion(instance.Metadata.GetEngine()),+ },+ })if err != nil {- return nil, errors.Wrapf(err, "failed to begin migration")+ return nil, errors.Wrapf(err, "failed to create changelog")}// Execute the SQL_, migrationErr := driver.Execute(driverCtx, sheet.Statement, opts)// Dump after migration and update changelog- if err := endMigration(- ctx, exec.store, exec.schemaSyncer, database,- needDump, changelogUID, migrationErr == nil, opts,- ); err != nil {- slog.Error("failed to end migration", log.BBError(err))+ update := &store.UpdateChangelogMessage{+ UID: changelogUID,+ }+ if needDump {+ opts.LogDatabaseSyncStart()+ syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)+ if err != nil {+ opts.LogDatabaseSyncEnd(err.Error())+ slog.Error("failed to sync database schema", log.BBError(err))+ } else {+ opts.LogDatabaseSyncEnd("")+ update.SyncHistoryUID = &syncHistory+ }+ }+ if migrationErr == nil {+ status := store.ChangelogStatusDone+ update.Status = &status+ } else {+ status := store.ChangelogStatusFailed+ update.Status = &status+ }+ if err := exec.store.UpdateChangelog(ctx, update); err != nil {+ slog.Error("failed to update changelog", log.BBError(err))}if migrationErr != nil {return nil, migrationErr}- // Post migration - clean up drift- slog.Debug("Post migration...",- slog.String("instance", instance.ResourceID),- slog.String("database", database.DatabaseName),- )-- if _, err := exec.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- MetadataUpdates: []func(*storepb.DatabaseMetadata){func(md *storepb.DatabaseMetadata) {- md.Drifted = false- }},- }); err != nil {- return nil, errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)- }-return &storepb.TaskRunResult{HasPriorBackup: priorBackupDetail != nil && len(priorBackupDetail.Items) > 0,}, nil@@ -278,9 +297,6 @@ func (exec *DatabaseMigrateExecutor) runStandardMigration(ctx context.Context, dfunc (exec *DatabaseMigrateExecutor) runGhostMigration(ctx context.Context, driverCtx context.Context, task *store.TaskMessage, taskRunUID int, sheet *store.SheetMessage, instance *store.InstanceMessage, database *store.DatabaseMessage, project *store.ProjectMessage) (*storepb.TaskRunResult, error) {flags := task.Payload.GetFlags()- needDump := computeNeedDump(task.Type, database.Engine, sheet.Statement)- taskRunName := common.FormatTaskRun(database.ProjectID, task.PlanID, task.Environment, task.ID, taskRunUID)-// Get database driverdriver, err := exec.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{TenantMode: project.Setting.GetPostgresDatabaseTenantMode(),@@ -333,14 +349,21 @@ func (exec *DatabaseMigrateExecutor) runGhostMigration(ctx context.Context, drivgomysql.DeregisterTLSConfig(migrationContext.Uuid)}()- // Begin migration - dump before migration- changelogUID, err := beginMigration(- ctx, exec.store, database,- taskRunName,- storepb.ChangelogPayload_MIGRATE, schema.GetDumpFormatVersion(instance.Metadata.GetEngine()), exec.profile.GitCommit,- )+ // Begin migration - create pending changelog+ changelogUID, err := exec.store.CreateChangelog(ctx, &store.ChangelogMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ Status: store.ChangelogStatusPending,+ SyncHistoryUID: nil,+ Payload: &storepb.ChangelogPayload{+ TaskRun: common.FormatTaskRun(database.ProjectID, task.PlanID, task.Environment, task.ID, taskRunUID),+ Type: storepb.ChangelogPayload_MIGRATE,+ GitCommit: exec.profile.GitCommit,+ DumpVersion: schema.GetDumpFormatVersion(instance.Metadata.GetEngine()),+ },+ })if err != nil {- return nil, errors.Wrapf(err, "failed to begin migration")+ return nil, errors.Wrapf(err, "failed to create changelog")}// Execute gh-ost migration@@ -385,33 +408,33 @@ func (exec *DatabaseMigrateExecutor) runGhostMigration(ctx context.Context, driv}// Dump after migration and update changelog- if err := endMigration(- ctx, exec.store, exec.schemaSyncer, database,- needDump, changelogUID, migrationErr == nil, opts,- ); err != nil {- slog.Error("failed to end migration", log.BBError(err))+ update := &store.UpdateChangelogMessage{+ UID: changelogUID,+ }+ opts.LogDatabaseSyncStart()+ syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)+ if err != nil {+ opts.LogDatabaseSyncEnd(err.Error())+ slog.Error("failed to sync database schema", log.BBError(err))+ } else {+ opts.LogDatabaseSyncEnd("")+ update.SyncHistoryUID = &syncHistory+ }+ if migrationErr == nil {+ status := store.ChangelogStatusDone+ update.Status = &status+ } else {+ status := store.ChangelogStatusFailed+ update.Status = &status+ }+ if err := exec.store.UpdateChangelog(ctx, update); err != nil {+ slog.Error("failed to update changelog", log.BBError(err))}if migrationErr != nil {return nil, migrationErr}- // Post migration - clean up drift- slog.Debug("Post migration...",- slog.String("instance", instance.ResourceID),- slog.String("database", database.DatabaseName),- )-- if _, err := exec.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- MetadataUpdates: []func(*storepb.DatabaseMetadata){func(md *storepb.DatabaseMetadata) {- md.Drifted = false- }},- }); err != nil {- return nil, errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)- }-return &storepb.TaskRunResult{}, nil}@@ -433,6 +456,47 @@ func (exec *DatabaseMigrateExecutor) runVersionedRelease(ctx context.Context, dr}}+ taskRunName := common.FormatTaskRun(database.ProjectID, task.PlanID, task.Environment, task.ID, taskRunUID)++ // Create pending changelog for the entire release+ changelogUID, err := exec.store.CreateChangelog(ctx, &store.ChangelogMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ Status: store.ChangelogStatusPending,+ SyncHistoryUID: nil,+ Payload: &storepb.ChangelogPayload{+ TaskRun: taskRunName,+ Type: storepb.ChangelogPayload_MIGRATE,+ GitCommit: exec.profile.GitCommit,+ DumpVersion: schema.GetDumpFormatVersion(instance.Metadata.GetEngine()),+ },+ })+ if err != nil {+ return nil, errors.Wrapf(err, "failed to create changelog")+ }++ // Set up execute options+ opts := db.ExecuteOptions{}+ if project != nil && project.Setting != nil {+ opts.MaximumRetries = int(project.Setting.GetExecutionRetryPolicy().GetMaximumRetries())+ }+ opts.CreateTaskRunLog = func(t time.Time, e *storepb.TaskRunLog) error {+ return exec.store.CreateTaskRunLog(ctx, taskRunUID, t.UTC(), exec.profile.DeployID, e)+ }++ // Get database driver once for all files+ driver, err := exec.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{+ TenantMode: project.Setting.GetPostgresDatabaseTenantMode(),+ TaskRunUID: &taskRunUID,+ })+ if err != nil {+ return nil, errors.Wrapf(err, "failed to get driver connection for instance %q", instance.ResourceID)+ }+ defer driver.Close(ctx)++ var migrationErr error+ var lastAppliedVersion string+// Execute unapplied files in orderfor _, file := range release.Payload.Files {// Skip if already applied@@ -465,66 +529,20 @@ func (exec *DatabaseMigrateExecutor) runVersionedRelease(ctx context.Context, dr},})- needDump := computeNeedDump(task.Type, database.Engine, sheet.Statement)- taskRunName := common.FormatTaskRun(database.ProjectID, task.PlanID, task.Environment, task.ID, taskRunUID)-- // Get database driver- driver, err := exec.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{- TenantMode: project.Setting.GetPostgresDatabaseTenantMode(),- TaskRunUID: &taskRunUID,- })- if err != nil {- return nil, errors.Wrapf(err, "failed to get driver connection for instance %q", instance.ResourceID)- }-slog.Debug("Start migration...",slog.String("instance", database.InstanceID),slog.String("database", database.DatabaseName),slog.String("type", task.Type.String()),)- // Set up execute options- opts := db.ExecuteOptions{}- if project != nil && project.Setting != nil {- opts.MaximumRetries = int(project.Setting.GetExecutionRetryPolicy().GetMaximumRetries())- }- opts.CreateTaskRunLog = func(t time.Time, e *storepb.TaskRunLog) error {- return exec.store.CreateTaskRunLog(ctx, taskRunUID, t.UTC(), exec.profile.DeployID, e)- }-- // Begin migration - dump before migration- changelogUID, err := beginMigration(- ctx, exec.store, database,- taskRunName,- storepb.ChangelogPayload_MIGRATE, schema.GetDumpFormatVersion(instance.Metadata.GetEngine()), exec.profile.GitCommit,- )- if err != nil {- driver.Close(ctx)- return nil, errors.Wrapf(err, "failed to begin migration for version %s", file.Version)- }-// Execute the SQL- _, migrationErr := driver.Execute(driverCtx, sheet.Statement, opts)-- // Dump after migration and update changelog- if err := endMigration(- ctx, exec.store, exec.schemaSyncer, database,- needDump, changelogUID, migrationErr == nil, opts,- ); err != nil {- slog.Error("failed to end migration", log.BBError(err))- }-- if migrationErr != nil {- driver.Close(ctx)- return nil, errors.Wrapf(migrationErr, "failed to execute release file %s (version %s)", file.Path, file.Version)+ _, err = driver.Execute(driverCtx, sheet.Statement, opts)+ if err != nil {+ migrationErr = errors.Wrapf(err, "failed to execute release file %s (version %s)", file.Path, file.Version)+ break}- // Post migration - create revision and update database- slog.Debug("Post migration...",- slog.String("instance", instance.ResourceID),- slog.String("database", database.DatabaseName),- )-+ // Create revision for this filer := &store.RevisionMessage{InstanceID: database.InstanceID,DatabaseName: database.DatabaseName,@@ -540,37 +558,53 @@ func (exec *DatabaseMigrateExecutor) runVersionedRelease(ctx context.Context, dr_, err = exec.store.CreateRevision(ctx, r)if err != nil {- driver.Close(ctx)- return nil, errors.Wrapf(err, "failed to create revision for version %s", file.Version)+ migrationErr = errors.Wrapf(err, "failed to create revision for version %s", file.Version)+ break}- // Update database metadata with the version only if the new version is greater- if shouldUpdateVersion(database.Metadata.Version, file.Version) {- if _, err := exec.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- MetadataUpdates: []func(*storepb.DatabaseMetadata){func(md *storepb.DatabaseMetadata) {- md.Version = file.Version- }},- }); err != nil {- driver.Close(ctx)- return nil, errors.Wrapf(err, "failed to update database metadata with version %s", file.Version)- }- }+ // Track the last successfully applied version+ lastAppliedVersion = file.Version+ }++ // Update changelog after all files are processed+ update := &store.UpdateChangelogMessage{+ UID: changelogUID,+ }+ opts.LogDatabaseSyncStart()+ syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)+ if err != nil {+ opts.LogDatabaseSyncEnd(err.Error())+ slog.Error("failed to sync database schema", log.BBError(err))+ } else {+ opts.LogDatabaseSyncEnd("")+ update.SyncHistoryUID = &syncHistory+ }+ if migrationErr == nil {+ status := store.ChangelogStatusDone+ update.Status = &status+ } else {+ status := store.ChangelogStatusFailed+ update.Status = &status+ }+ if err := exec.store.UpdateChangelog(ctx, update); err != nil {+ slog.Error("failed to update changelog", log.BBError(err))+ }- // Clean up drift+ if migrationErr != nil {+ return nil, migrationErr+ }++ // Update database version to the last successfully applied version+ if lastAppliedVersion != "" && shouldUpdateVersion(database.Metadata.Version, lastAppliedVersion) {if _, err := exec.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{InstanceID: database.InstanceID,DatabaseName: database.DatabaseName,MetadataUpdates: []func(*storepb.DatabaseMetadata){func(md *storepb.DatabaseMetadata) {- md.Drifted = false+ md.Version = lastAppliedVersion}},}); err != nil {- driver.Close(ctx)- return nil, errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)+ return nil, errors.Wrapf(err, "failed to update database version to %s", lastAppliedVersion)}-- driver.Close(ctx)}return &storepb.TaskRunResult{}, nil@@ -610,9 +644,6 @@ func (exec *DatabaseMigrateExecutor) runDeclarativeRelease(ctx context.Context,},})- needDump := computeNeedDump(task.Type, database.Engine, sheet.Statement)- taskRunName := common.FormatTaskRun(database.ProjectID, task.PlanID, task.Environment, task.ID, taskRunUID)-// Get database driverdriver, err := exec.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{TenantMode: project.Setting.GetPostgresDatabaseTenantMode(),@@ -648,14 +679,21 @@ func (exec *DatabaseMigrateExecutor) runDeclarativeRelease(ctx context.Context,}opts.LogComputeDiffEnd("")- // Begin migration - dump before migration- changelogUID, err := beginMigration(- ctx, exec.store, database,- taskRunName,- storepb.ChangelogPayload_SDL, schema.GetDumpFormatVersion(instance.Metadata.GetEngine()), exec.profile.GitCommit,- )+ // Begin migration - create pending changelog+ changelogUID, err := exec.store.CreateChangelog(ctx, &store.ChangelogMessage{+ InstanceID: database.InstanceID,+ DatabaseName: database.DatabaseName,+ Status: store.ChangelogStatusPending,+ SyncHistoryUID: nil,+ Payload: &storepb.ChangelogPayload{+ TaskRun: common.FormatTaskRun(database.ProjectID, task.PlanID, task.Environment, task.ID, taskRunUID),+ Type: storepb.ChangelogPayload_SDL,+ GitCommit: exec.profile.GitCommit,+ DumpVersion: schema.GetDumpFormatVersion(instance.Metadata.GetEngine()),+ },+ })if err != nil {- return nil, errors.Wrapf(err, "failed to begin migration")+ return nil, errors.Wrapf(err, "failed to create changelog")}// Execute SDL migration@@ -664,35 +702,43 @@ func (exec *DatabaseMigrateExecutor) runDeclarativeRelease(ctx context.Context,_, migrationErr := driver.Execute(driverCtx, migrationSQL, opts)// Dump after migration and update changelog- if err := endMigration(- ctx, exec.store, exec.schemaSyncer, database,- needDump, changelogUID, migrationErr == nil, opts,- ); err != nil {- slog.Error("failed to end migration", log.BBError(err))+ update := &store.UpdateChangelogMessage{+ UID: changelogUID,+ }+ opts.LogDatabaseSyncStart()+ syncHistory, err := exec.schemaSyncer.SyncDatabaseSchemaToHistory(ctx, database)+ if err != nil {+ opts.LogDatabaseSyncEnd(err.Error())+ slog.Error("failed to sync database schema", log.BBError(err))+ } else {+ opts.LogDatabaseSyncEnd("")+ update.SyncHistoryUID = &syncHistory+ }+ if migrationErr == nil {+ status := store.ChangelogStatusDone+ update.Status = &status+ } else {+ status := store.ChangelogStatusFailed+ update.Status = &status+ }+ if err := exec.store.UpdateChangelog(ctx, update); err != nil {+ slog.Error("failed to update changelog", log.BBError(err))}if migrationErr != nil {return nil, errors.Wrap(migrationErr, "failed to execute declarative release")}- // Post migration+ // Post migration - update database schema version// Note: Declarative releases do NOT create revisions (they are version-tracked through the database schema itself)- slog.Debug("Post migration...",- slog.String("instance", instance.ResourceID),- slog.String("database", database.DatabaseName),- )-- // Clean up drift- // Update database schema version.if _, err := exec.store.UpdateDatabase(ctx, &store.UpdateDatabaseMessage{InstanceID: database.InstanceID,DatabaseName: database.DatabaseName,MetadataUpdates: []func(*storepb.DatabaseMetadata){func(md *storepb.DatabaseMetadata) {- md.Drifted = falsemd.Version = file.Version}},}); err != nil {- return nil, errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)+ return nil, errors.Wrapf(err, "failed to update database version for %q", database.DatabaseName)}return &storepb.TaskRunResult{}, nil@@ -1166,77 +1212,6 @@ func getPreviousSuccessfulSDLAndSchema(ctx context.Context, s *store.Store, instreturn previousUserSDLText, previousSchema, nil}-// beginMigration inserts a migration history record with pending status.-// Returns (changelogUID, error).-func beginMigration(- ctx context.Context,- stores *store.Store,- database *store.DatabaseMessage,- taskRunName string,- changelogType storepb.ChangelogPayload_Type,- dumpVersion int32,- gitCommit string,-) (int64, error) {- // create pending changelog- changelogUID, err := stores.CreateChangelog(ctx, &store.ChangelogMessage{- InstanceID: database.InstanceID,- DatabaseName: database.DatabaseName,- Status: store.ChangelogStatusPending,- SyncHistoryUID: nil,- Payload: &storepb.ChangelogPayload{- TaskRun: taskRunName,- Type: changelogType,- GitCommit: gitCommit,- DumpVersion: dumpVersion,- }})- if err != nil {- return 0, errors.Wrapf(err, "failed to create changelog")- }-- return changelogUID, nil-}--// endMigration updates the migration history record to DONE or FAILED depending on migration is done or not.-func endMigration(- ctx context.Context,- storeInstance *store.Store,- syncer *schemasync.Syncer,- database *store.DatabaseMessage,- needDump bool,- changelogUID int64,- isDone bool,- opts db.ExecuteOptions,-) error {- update := &store.UpdateChangelogMessage{- UID: changelogUID,- }-- if needDump {- opts.LogDatabaseSyncStart()- syncHistory, err := syncer.SyncDatabaseSchemaToHistory(ctx, database)- if err != nil {- opts.LogDatabaseSyncEnd(err.Error())- return errors.Wrapf(err, "failed to sync database metadata and schema")- }- opts.LogDatabaseSyncEnd("")- update.SyncHistoryUID = &syncHistory- }-- if isDone {- status := store.ChangelogStatusDone- update.Status = &status- } else {- status := store.ChangelogStatusFailed- update.Status = &status- }-- if err := storeInstance.UpdateChangelog(ctx, update); err != nil {- return errors.Wrapf(err, "failed to update changelog")- }-- return nil-}-// shouldUpdateVersion checks if newVersion is greater than currentVersion.// Returns true if:// - currentVersion is emptydiff --git a/backend/tests/action_test.go b/backend/tests/action_test.goindex f2fc438917..51b131f960 100644--- a/backend/tests/action_test.go+++ b/backend/tests/action_test.go@@ -963,11 +963,23 @@ func TestActionRolloutCommand(t *testing.T) {a.Equal("00003", updatedDatabase.Msg.SchemaVersion)// 6. Verify change history was recorded+ // Note: Versioned releases create one MIGRATE changelog for the entire release (not per file)+ // Plus a BASELINE changelog if this is the first migration for the databasechangelogs, err := ctl.databaseServiceClient.ListChangelogs(ctx, connect.NewRequest(&v1pb.ListChangelogsRequest{Parent: database.Name,}))a.NoError(err)- a.GreaterOrEqual(len(changelogs.Msg.Changelogs), 3, "Expected at least 3 migration records in history")+ // Expect at least 1 changelog (MIGRATE), possibly 2 if BASELINE was created+ a.GreaterOrEqual(len(changelogs.Msg.Changelogs), 1, "Expected at least 1 changelog for the versioned release")+ // Verify at least one MIGRATE changelog exists+ foundMigrateChangelog := false+ for _, changelog := range changelogs.Msg.Changelogs {+ if changelog.Type == v1pb.Changelog_MIGRATE {+ foundMigrateChangelog = true+ break+ }+ }+ a.True(foundMigrateChangelog, "Expected to find at least one MIGRATE changelog")revisions, err := ctl.revisionServiceClient.ListRevisions(ctx, connect.NewRequest(&v1pb.ListRevisionsRequest{Parent: database.Name,