Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: deprecate db_schema.todo column and column default migrator (#17869)
Danny Xu49d33a18a3a2
diff --git a/backend/common/engine.go b/backend/common/engine.goindex d0caefe36e..ff9f4ac304 100644--- a/backend/common/engine.go+++ b/backend/common/engine.go@@ -430,51 +430,6 @@ func BackupDatabaseNameOfEngine(e storepb.Engine) string {}}-// EngineDBSchemaReadyToMigrate returns true if the engine needs column default migration.-// This is used by both the migrator and the sync process to determine if a database-// needs the column default migration.-// When an engine's sync.go is updated to write to the Default field with proper-// normalization (like schema qualification), we move it to the false case.-func EngineDBSchemaReadyToMigrate(e storepb.Engine) bool {- //exhaustive:enforce- switch e {- case- storepb.Engine_POSTGRES,- storepb.Engine_MYSQL,- storepb.Engine_MSSQL,- storepb.Engine_ORACLE,- storepb.Engine_MARIADB,- storepb.Engine_OCEANBASE,- storepb.Engine_SNOWFLAKE,- storepb.Engine_CLICKHOUSE,- storepb.Engine_COCKROACHDB,- storepb.Engine_SPANNER,- storepb.Engine_BIGQUERY,- storepb.Engine_REDSHIFT,- storepb.Engine_STARROCKS,- storepb.Engine_DORIS:- // These engines have been migrated to use the Default field- return true- case- storepb.Engine_ENGINE_UNSPECIFIED,- storepb.Engine_CASSANDRA,- storepb.Engine_SQLITE,- storepb.Engine_MONGODB,- storepb.Engine_REDIS,- storepb.Engine_HIVE,- storepb.Engine_DYNAMODB,- storepb.Engine_ELASTICSEARCH,- storepb.Engine_DATABRICKS,- storepb.Engine_COSMOSDB,- storepb.Engine_TIDB,- storepb.Engine_TRINO:- // These engines don't have traditional column defaults or are NoSQL databases.- return true- default:- return true- }-}-// TransactionMode represents the transaction execution mode for a migration script.type TransactionMode stringdiff --git a/backend/migrator/migration/3.11/0019##drop_db_schema_todo.sql b/backend/migrator/migration/3.11/0019##drop_db_schema_todo.sqlnew file mode 100644index 0000000000..f26a6a428f--- /dev/null+++ b/backend/migrator/migration/3.11/0019##drop_db_schema_todo.sql@@ -0,0 +1 @@+ALTER TABLE db_schema DROP COLUMN todo;diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sqlindex aa7d731765..3cfc68b469 100644--- a/backend/migrator/migration/LATEST.sql+++ b/backend/migrator/migration/LATEST.sql@@ -159,7 +159,6 @@ CREATE TABLE db_schema (raw_dump text NOT NULL DEFAULT '',-- Stored as DatabaseConfig (proto/store/store/database.proto)config jsonb NOT NULL DEFAULT '{}',- todo boolean NOT NULL DEFAULT TRUE,CONSTRAINT db_schema_instance_db_name_fkey FOREIGN KEY(instance, db_name) REFERENCES db(instance, name));diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.goindex 73c3ceecad..a024162cce 100644--- a/backend/migrator/migrator_test.go+++ b/backend/migrator/migrator_test.go@@ -12,7 +12,7 @@ import (func TestLatestVersion(t *testing.T) {files, err := getSortedVersionedFiles()require.NoError(t, err)- require.Equal(t, semver.MustParse("3.11.18"), *files[len(files)-1].version)+ require.Equal(t, semver.MustParse("3.11.19"), *files[len(files)-1].version)}func TestVersionUnique(t *testing.T) {diff --git a/backend/runner/migrator/column_default_migrator.go b/backend/runner/migrator/column_default_migrator.godeleted file mode 100644index 82b0877a8f..0000000000--- a/backend/runner/migrator/column_default_migrator.go+++ /dev/null@@ -1,158 +0,0 @@-package migrator--import (- "context"- "fmt"- "log/slog"- "sync"- "time"-- "google.golang.org/protobuf/encoding/protojson"-- "github.com/bytebase/bytebase/backend/common"- "github.com/bytebase/bytebase/backend/common/log"- storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/db/mysql"- "github.com/bytebase/bytebase/backend/store"-)--const (- migrationInterval = 10 * time.Second- batchSize = 20-)--// ColumnDefaultMigrator is the migrator for column default values.-// It will migrate the `default_null` and `default_expression` to `default`.-// The process is asynchronous and will not block the application startup.-// The migrator will be started after the server is started.-type ColumnDefaultMigrator struct {- store *store.Store- supportedEngines []storepb.Engine-}--// NewColumnDefaultMigrator creates a new ColumnDefaultMigrator.-func NewColumnDefaultMigrator(store *store.Store, supportedEngines []storepb.Engine) *ColumnDefaultMigrator {- return &ColumnDefaultMigrator{- store: store,- supportedEngines: supportedEngines,- }-}--// EnginesNeedingMigration returns the list of engines that currently need column default migration.-// This function dynamically builds the list based on common.EngineNeedsColumnDefaultMigration.-func EnginesNeedingMigration() []storepb.Engine {- var engines []storepb.Engine- // Check all known engines- for _, engine := range storepb.Engine_value {- if common.EngineDBSchemaReadyToMigrate(storepb.Engine(engine)) {- engines = append(engines, storepb.Engine(engine))- }- }- return engines-}--// Run starts the ColumnDefaultMigrator.-func (m *ColumnDefaultMigrator) Run(ctx context.Context, wg *sync.WaitGroup) {- defer wg.Done()- ticker := time.NewTicker(migrationInterval)- defer ticker.Stop()- for {- select {- case <-ticker.C:- if err := m.migrate(ctx); err != nil {- slog.Error("Failed to run column default value migrator", log.BBError(err))- }- case <-ctx.Done():- return- }- }-}--func (m *ColumnDefaultMigrator) migrate(ctx context.Context) error {- for _, engine := range m.supportedEngines {- dbSchemas, err := m.store.ListDBSchemasWithTodo(ctx, engine, batchSize)- if err != nil {- return err- }-- if len(dbSchemas) == 0 {- continue- }-- for _, dbSchema := range dbSchemas {- metadata := &storepb.DatabaseSchemaMetadata{}- if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(dbSchema.Metadata), metadata); err != nil {- slog.Warn("Failed to unmarshal db schema metadata", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))- // We mark the row as done even if it fails to unmarshal, to avoid retrying forever.- if err := m.store.UpdateDBSchemaTodo(ctx, dbSchema.ID, false); err != nil {- slog.Error("Failed to mark db schema as done", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))- }- continue- }-- changed := false- for _, schema := range metadata.Schemas {- for _, table := range schema.Tables {- for _, column := range table.Columns {- switch engine {- case storepb.Engine_MYSQL, storepb.Engine_TIDB:- if column.DefaultNull {- column.Default = "NULL"- column.DefaultNull = false- changed = true- } else if column.Default != "" {- column.Default = fmt.Sprintf("'%s'", column.Default)- changed = true- } else if column.DefaultExpression != "" {- if mysql.IsCurrentTimestampLike(column.DefaultExpression) {- column.Default = column.DefaultExpression- column.DefaultExpression = ""- } else {- quotedExpression := fmt.Sprintf("'%s'", column.DefaultExpression)- unquotedExpression := mysql.UnquoteMySQLString(quotedExpression)- column.Default = mysql.UnescapeExpressionDefault(unquotedExpression)- column.DefaultExpression = ""- }- changed = true- }- default:- // For Oracle, PostgreSQL, MSSQL and other engines: simple migration- if column.DefaultNull {- column.Default = "NULL"- column.DefaultNull = false- changed = true- } else if column.DefaultExpression != "" {- column.Default = column.DefaultExpression- column.DefaultExpression = ""- changed = true- }- }- }- }- }-- // If no changes were made, just mark as done.- if !changed {- if err := m.store.UpdateDBSchemaTodo(ctx, dbSchema.ID, false); err != nil {- slog.Error("Failed to mark db schema as done", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))- }- continue- }-- marshaled, err := protojson.Marshal(metadata)- if err != nil {- slog.Error("Failed to marshal db schema metadata", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))- continue- }-- // Update metadata and todo in a single transaction, only if todo is still true.- // This prevents race conditions with the sync process.- if err := m.store.UpdateDBSchemaMetadataIfTodo(ctx, dbSchema.ID, string(marshaled)); err != nil {- slog.Error("Failed to update db schema metadata and todo", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))- continue- }- }- }-- return nil-}diff --git a/backend/runner/schemasync/syncer.go b/backend/runner/schemasync/syncer.goindex b39d6fb15b..897e911674 100644--- a/backend/runner/schemasync/syncer.go+++ b/backend/runner/schemasync/syncer.go@@ -451,10 +451,9 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Databreturn 0, errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)}- todo := !common.EngineDBSchemaReadyToMigrate(instance.Metadata.GetEngine())if err := s.store.UpsertDBSchema(ctx,database.InstanceID, database.DatabaseName,- databaseMetadata, dbConfig, rawDump, todo,+ databaseMetadata, dbConfig, rawDump,); err != nil {if strings.Contains(err.Error(), "escape sequence") {if metadataBytes, err := protojson.Marshal(databaseMetadata); err == nil {diff --git a/backend/server/server.go b/backend/server/server.goindex 946bc51d26..bfce7a4de6 100644--- a/backend/server/server.go+++ b/backend/server/server.go@@ -52,14 +52,13 @@ const (// Server is the Bytebase server.type Server struct {// Asynchronous runners.- taskSchedulerV2 *taskrun.SchedulerV2- planCheckScheduler *plancheck.Scheduler- metricReporter *metricreport.Reporter- schemaSyncer *schemasync.Syncer- approvalRunner *approval.Runner- columnDefaultMigrator *runnermigrator.ColumnDefaultMigrator- exportArchiveCleaner *runnermigrator.ExportArchiveCleaner- runnerWG sync.WaitGroup+ taskSchedulerV2 *taskrun.SchedulerV2+ planCheckScheduler *plancheck.Scheduler+ metricReporter *metricreport.Reporter+ schemaSyncer *schemasync.Syncer+ approvalRunner *approval.Runner+ exportArchiveCleaner *runnermigrator.ExportArchiveCleaner+ runnerWG sync.WaitGroupwebhookManager *webhook.ManageriamManager *iam.Manager@@ -204,9 +203,6 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {statementReportExecutor := plancheck.NewStatementReportExecutor(stores, sheetManager, s.dbFactory)s.planCheckScheduler.Register(store.PlanCheckDatabaseStatementSummaryReport, statementReportExecutor)- // Column default value migrator- s.columnDefaultMigrator = runnermigrator.NewColumnDefaultMigrator(stores, runnermigrator.EnginesNeedingMigration())-// Export archive cleaners.exportArchiveCleaner = runnermigrator.NewExportArchiveCleaner(stores)@@ -245,9 +241,6 @@ func (s *Server) Run(ctx context.Context, port int) error {s.runnerWG.Add(1)go s.planCheckScheduler.Run(ctx, &s.runnerWG)- s.runnerWG.Add(1)- go s.columnDefaultMigrator.Run(ctx, &s.runnerWG)-s.runnerWG.Add(1)go s.exportArchiveCleaner.Run(ctx, &s.runnerWG)diff --git a/backend/store/db_schema.go b/backend/store/db_schema.goindex e47639dc9a..3bdfbb7150 100644--- a/backend/store/db_schema.go+++ b/backend/store/db_schema.go@@ -80,7 +80,6 @@ func (s *Store) UpsertDBSchema(dbMetadata *storepb.DatabaseSchemaMetadata,dbConfig *storepb.DatabaseConfig,dbSchema []byte,- todo bool,) error {metadataBytes, err := protojson.Marshal(dbMetadata)if err != nil {@@ -97,23 +96,20 @@ func (s *Store) UpsertDBSchema(db_name,metadata,raw_dump,- config,- todo+ config)- VALUES (?, ?, ?, ?, ?, ?)+ VALUES (?, ?, ?, ?, ?)ON CONFLICT(instance, db_name) DO UPDATE SETmetadata = EXCLUDED.metadata,raw_dump = EXCLUDED.raw_dump,- config = EXCLUDED.config,- todo = EXCLUDED.todo+ config = EXCLUDED.configRETURNING metadata, raw_dump, config`,instanceID,databaseName,metadataBytes,// Convert to string because []byte{} is null which violates db schema constraints.string(dbSchema),- configBytes,- todo)+ configBytes)query, args, err := q.ToSQL()if err != nil {@@ -183,138 +179,6 @@ func (s *Store) UpdateDBSchema(ctx context.Context, instanceID, databaseName strreturn nil}-// DBSchemaWithTodo is a struct for a db_schema with the todo column.-// It is used for the column default value migration.-type DBSchemaWithTodo struct {- ID int- InstanceID string- DBName string- Metadata string-}--// ListDBSchemasWithTodo lists all db_schemas with todo = true.-func (s *Store) ListDBSchemasWithTodo(ctx context.Context, engineType storepb.Engine, limit int) ([]*DBSchemaWithTodo, error) {- q := qb.Q().Space(`- SELECT- db_schema.id,- db_schema.instance,- db_schema.db_name,- db_schema.metadata- FROM db_schema- LEFT JOIN instance ON db_schema.instance = instance.resource_id- WHERE db_schema.todo = true AND instance.metadata->>'engine' = ?- LIMIT ?`, engineType.String(), limit)-- query, args, err := q.ToSQL()- if err != nil {- return nil, errors.Wrapf(err, "failed to build sql")- }-- rows, err := s.GetDB().QueryContext(ctx, query, args...)- if err != nil {- return nil, err- }- defer rows.Close()-- var dbSchemas []*DBSchemaWithTodo- for rows.Next() {- var dbSchema DBSchemaWithTodo- if err := rows.Scan(- &dbSchema.ID,- &dbSchema.InstanceID,- &dbSchema.DBName,- &dbSchema.Metadata,- ); err != nil {- return nil, err- }- dbSchemas = append(dbSchemas, &dbSchema)- }- if err := rows.Err(); err != nil {- return nil, err- }-- return dbSchemas, nil-}--// UpdateDBSchemaTodo updates the todo column of a db_schema.-func (s *Store) UpdateDBSchemaTodo(ctx context.Context, id int, todo bool) error {- q := qb.Q().Space("UPDATE db_schema SET todo = ? WHERE id = ?", todo, id)-- query, args, err := q.ToSQL()- if err != nil {- return errors.Wrapf(err, "failed to build sql")- }-- tx, err := s.GetDB().BeginTx(ctx, nil)- if err != nil {- return err- }- defer tx.Rollback()-- if _, err := tx.ExecContext(ctx, query, args...); err != nil {- return err- }-- return tx.Commit()-}--// UpdateDBSchemaMetadata updates the metadata of a db_schema.-func (s *Store) UpdateDBSchemaMetadata(ctx context.Context, id int, metadata string) error {- q := qb.Q().Space("UPDATE db_schema SET metadata = ? WHERE id = ?", metadata, id)-- query, args, err := q.ToSQL()- if err != nil {- return errors.Wrapf(err, "failed to build sql")- }-- tx, err := s.GetDB().BeginTx(ctx, nil)- if err != nil {- return err- }- defer tx.Rollback()-- if _, err := tx.ExecContext(ctx, query, args...); err != nil {- return err- }-- return tx.Commit()-}--// UpdateDBSchemaMetadataIfTodo updates the metadata and sets todo to false only if todo is currently true.-// This is used by the migrator to avoid race conditions with the sync process.-// The WHERE condition ensures atomic check-and-update, preventing any race conditions.-func (s *Store) UpdateDBSchemaMetadataIfTodo(ctx context.Context, id int, metadata string) error {- q := qb.Q().Space("UPDATE db_schema SET metadata = ?, todo = false WHERE id = ? AND todo = true", metadata, id)-- query, args, err := q.ToSQL()- if err != nil {- return errors.Wrapf(err, "failed to build sql")- }-- tx, err := s.GetDB().BeginTx(ctx, nil)- if err != nil {- return err- }- defer tx.Rollback()-- result, err := tx.ExecContext(ctx, query, args...)- if err != nil {- return err- }-- rowsAffected, err := result.RowsAffected()- if err != nil {- return err- }-- if rowsAffected == 0 {- // Schema was already processed by sync, skip update- return tx.Rollback()- }-- return tx.Commit()-}-func (s *Store) convertMetadataAndConfig(ctx context.Context, metadata, schema, config []byte, instanceID string) (*model.DatabaseSchema, error) {var databaseSchema storepb.DatabaseSchemaMetadatavar databaseConfig storepb.DatabaseConfig