Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

chore: deprecate db_schema.todo column and column default migrator (#17869)

Danny Xu49d33a18a3a2
diff --git a/backend/common/engine.go b/backend/common/engine.go
index d0caefe36e..ff9f4ac304 100644
--- a/backend/common/engine.go
+++ b/backend/common/engine.go
@@ -430,51 +430,6 @@ func BackupDatabaseNameOfEngine(e storepb.Engine) string {
}
}
-// EngineDBSchemaReadyToMigrate returns true if the engine needs column default migration.
-// This is used by both the migrator and the sync process to determine if a database
-// needs the column default migration.
-// When an engine's sync.go is updated to write to the Default field with proper
-// normalization (like schema qualification), we move it to the false case.
-func EngineDBSchemaReadyToMigrate(e storepb.Engine) bool {
- //exhaustive:enforce
- switch e {
- case
- storepb.Engine_POSTGRES,
- storepb.Engine_MYSQL,
- storepb.Engine_MSSQL,
- storepb.Engine_ORACLE,
- storepb.Engine_MARIADB,
- storepb.Engine_OCEANBASE,
- storepb.Engine_SNOWFLAKE,
- storepb.Engine_CLICKHOUSE,
- storepb.Engine_COCKROACHDB,
- storepb.Engine_SPANNER,
- storepb.Engine_BIGQUERY,
- storepb.Engine_REDSHIFT,
- storepb.Engine_STARROCKS,
- storepb.Engine_DORIS:
- // These engines have been migrated to use the Default field
- return true
- case
- storepb.Engine_ENGINE_UNSPECIFIED,
- storepb.Engine_CASSANDRA,
- storepb.Engine_SQLITE,
- storepb.Engine_MONGODB,
- storepb.Engine_REDIS,
- storepb.Engine_HIVE,
- storepb.Engine_DYNAMODB,
- storepb.Engine_ELASTICSEARCH,
- storepb.Engine_DATABRICKS,
- storepb.Engine_COSMOSDB,
- storepb.Engine_TIDB,
- storepb.Engine_TRINO:
- // These engines don't have traditional column defaults or are NoSQL databases.
- return true
- default:
- return true
- }
-}
-
// TransactionMode represents the transaction execution mode for a migration script.
type TransactionMode string
diff --git a/backend/migrator/migration/3.11/0019##drop_db_schema_todo.sql b/backend/migrator/migration/3.11/0019##drop_db_schema_todo.sql
new file mode 100644
index 0000000000..f26a6a428f
--- /dev/null
+++ b/backend/migrator/migration/3.11/0019##drop_db_schema_todo.sql
@@ -0,0 +1 @@
+ALTER TABLE db_schema DROP COLUMN todo;
diff --git a/backend/migrator/migration/LATEST.sql b/backend/migrator/migration/LATEST.sql
index aa7d731765..3cfc68b469 100644
--- a/backend/migrator/migration/LATEST.sql
+++ b/backend/migrator/migration/LATEST.sql
@@ -159,7 +159,6 @@ CREATE TABLE db_schema (
raw_dump text NOT NULL DEFAULT '',
-- Stored as DatabaseConfig (proto/store/store/database.proto)
config jsonb NOT NULL DEFAULT '{}',
- todo boolean NOT NULL DEFAULT TRUE,
CONSTRAINT db_schema_instance_db_name_fkey FOREIGN KEY(instance, db_name) REFERENCES db(instance, name)
);
diff --git a/backend/migrator/migrator_test.go b/backend/migrator/migrator_test.go
index 73c3ceecad..a024162cce 100644
--- a/backend/migrator/migrator_test.go
+++ b/backend/migrator/migrator_test.go
@@ -12,7 +12,7 @@ import (
func TestLatestVersion(t *testing.T) {
files, err := getSortedVersionedFiles()
require.NoError(t, err)
- require.Equal(t, semver.MustParse("3.11.18"), *files[len(files)-1].version)
+ require.Equal(t, semver.MustParse("3.11.19"), *files[len(files)-1].version)
}
func TestVersionUnique(t *testing.T) {
diff --git a/backend/runner/migrator/column_default_migrator.go b/backend/runner/migrator/column_default_migrator.go
deleted file mode 100644
index 82b0877a8f..0000000000
--- a/backend/runner/migrator/column_default_migrator.go
+++ /dev/null
@@ -1,158 +0,0 @@
-package migrator
-
-import (
- "context"
- "fmt"
- "log/slog"
- "sync"
- "time"
-
- "google.golang.org/protobuf/encoding/protojson"
-
- "github.com/bytebase/bytebase/backend/common"
- "github.com/bytebase/bytebase/backend/common/log"
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/db/mysql"
- "github.com/bytebase/bytebase/backend/store"
-)
-
-const (
- migrationInterval = 10 * time.Second
- batchSize = 20
-)
-
-// ColumnDefaultMigrator is the migrator for column default values.
-// It will migrate the `default_null` and `default_expression` to `default`.
-// The process is asynchronous and will not block the application startup.
-// The migrator will be started after the server is started.
-type ColumnDefaultMigrator struct {
- store *store.Store
- supportedEngines []storepb.Engine
-}
-
-// NewColumnDefaultMigrator creates a new ColumnDefaultMigrator.
-func NewColumnDefaultMigrator(store *store.Store, supportedEngines []storepb.Engine) *ColumnDefaultMigrator {
- return &ColumnDefaultMigrator{
- store: store,
- supportedEngines: supportedEngines,
- }
-}
-
-// EnginesNeedingMigration returns the list of engines that currently need column default migration.
-// This function dynamically builds the list based on common.EngineNeedsColumnDefaultMigration.
-func EnginesNeedingMigration() []storepb.Engine {
- var engines []storepb.Engine
- // Check all known engines
- for _, engine := range storepb.Engine_value {
- if common.EngineDBSchemaReadyToMigrate(storepb.Engine(engine)) {
- engines = append(engines, storepb.Engine(engine))
- }
- }
- return engines
-}
-
-// Run starts the ColumnDefaultMigrator.
-func (m *ColumnDefaultMigrator) Run(ctx context.Context, wg *sync.WaitGroup) {
- defer wg.Done()
- ticker := time.NewTicker(migrationInterval)
- defer ticker.Stop()
- for {
- select {
- case <-ticker.C:
- if err := m.migrate(ctx); err != nil {
- slog.Error("Failed to run column default value migrator", log.BBError(err))
- }
- case <-ctx.Done():
- return
- }
- }
-}
-
-func (m *ColumnDefaultMigrator) migrate(ctx context.Context) error {
- for _, engine := range m.supportedEngines {
- dbSchemas, err := m.store.ListDBSchemasWithTodo(ctx, engine, batchSize)
- if err != nil {
- return err
- }
-
- if len(dbSchemas) == 0 {
- continue
- }
-
- for _, dbSchema := range dbSchemas {
- metadata := &storepb.DatabaseSchemaMetadata{}
- if err := common.ProtojsonUnmarshaler.Unmarshal([]byte(dbSchema.Metadata), metadata); err != nil {
- slog.Warn("Failed to unmarshal db schema metadata", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))
- // We mark the row as done even if it fails to unmarshal, to avoid retrying forever.
- if err := m.store.UpdateDBSchemaTodo(ctx, dbSchema.ID, false); err != nil {
- slog.Error("Failed to mark db schema as done", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))
- }
- continue
- }
-
- changed := false
- for _, schema := range metadata.Schemas {
- for _, table := range schema.Tables {
- for _, column := range table.Columns {
- switch engine {
- case storepb.Engine_MYSQL, storepb.Engine_TIDB:
- if column.DefaultNull {
- column.Default = "NULL"
- column.DefaultNull = false
- changed = true
- } else if column.Default != "" {
- column.Default = fmt.Sprintf("'%s'", column.Default)
- changed = true
- } else if column.DefaultExpression != "" {
- if mysql.IsCurrentTimestampLike(column.DefaultExpression) {
- column.Default = column.DefaultExpression
- column.DefaultExpression = ""
- } else {
- quotedExpression := fmt.Sprintf("'%s'", column.DefaultExpression)
- unquotedExpression := mysql.UnquoteMySQLString(quotedExpression)
- column.Default = mysql.UnescapeExpressionDefault(unquotedExpression)
- column.DefaultExpression = ""
- }
- changed = true
- }
- default:
- // For Oracle, PostgreSQL, MSSQL and other engines: simple migration
- if column.DefaultNull {
- column.Default = "NULL"
- column.DefaultNull = false
- changed = true
- } else if column.DefaultExpression != "" {
- column.Default = column.DefaultExpression
- column.DefaultExpression = ""
- changed = true
- }
- }
- }
- }
- }
-
- // If no changes were made, just mark as done.
- if !changed {
- if err := m.store.UpdateDBSchemaTodo(ctx, dbSchema.ID, false); err != nil {
- slog.Error("Failed to mark db schema as done", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))
- }
- continue
- }
-
- marshaled, err := protojson.Marshal(metadata)
- if err != nil {
- slog.Error("Failed to marshal db schema metadata", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))
- continue
- }
-
- // Update metadata and todo in a single transaction, only if todo is still true.
- // This prevents race conditions with the sync process.
- if err := m.store.UpdateDBSchemaMetadataIfTodo(ctx, dbSchema.ID, string(marshaled)); err != nil {
- slog.Error("Failed to update db schema metadata and todo", slog.Int("db_schema_id", dbSchema.ID), slog.String("db_name", dbSchema.DBName), log.BBError(err))
- continue
- }
- }
- }
-
- return nil
-}
diff --git a/backend/runner/schemasync/syncer.go b/backend/runner/schemasync/syncer.go
index b39d6fb15b..897e911674 100644
--- a/backend/runner/schemasync/syncer.go
+++ b/backend/runner/schemasync/syncer.go
@@ -451,10 +451,9 @@ func (s *Syncer) doSyncDatabaseSchema(ctx context.Context, database *store.Datab
return 0, errors.Wrapf(err, "failed to update database %q for instance %q", database.DatabaseName, database.InstanceID)
}
- todo := !common.EngineDBSchemaReadyToMigrate(instance.Metadata.GetEngine())
if err := s.store.UpsertDBSchema(ctx,
database.InstanceID, database.DatabaseName,
- databaseMetadata, dbConfig, rawDump, todo,
+ databaseMetadata, dbConfig, rawDump,
); err != nil {
if strings.Contains(err.Error(), "escape sequence") {
if metadataBytes, err := protojson.Marshal(databaseMetadata); err == nil {
diff --git a/backend/server/server.go b/backend/server/server.go
index 946bc51d26..bfce7a4de6 100644
--- a/backend/server/server.go
+++ b/backend/server/server.go
@@ -52,14 +52,13 @@ const (
// Server is the Bytebase server.
type Server struct {
// Asynchronous runners.
- taskSchedulerV2 *taskrun.SchedulerV2
- planCheckScheduler *plancheck.Scheduler
- metricReporter *metricreport.Reporter
- schemaSyncer *schemasync.Syncer
- approvalRunner *approval.Runner
- columnDefaultMigrator *runnermigrator.ColumnDefaultMigrator
- exportArchiveCleaner *runnermigrator.ExportArchiveCleaner
- runnerWG sync.WaitGroup
+ taskSchedulerV2 *taskrun.SchedulerV2
+ planCheckScheduler *plancheck.Scheduler
+ metricReporter *metricreport.Reporter
+ schemaSyncer *schemasync.Syncer
+ approvalRunner *approval.Runner
+ exportArchiveCleaner *runnermigrator.ExportArchiveCleaner
+ runnerWG sync.WaitGroup
webhookManager *webhook.Manager
iamManager *iam.Manager
@@ -204,9 +203,6 @@ func NewServer(ctx context.Context, profile *config.Profile) (*Server, error) {
statementReportExecutor := plancheck.NewStatementReportExecutor(stores, sheetManager, s.dbFactory)
s.planCheckScheduler.Register(store.PlanCheckDatabaseStatementSummaryReport, statementReportExecutor)
- // Column default value migrator
- s.columnDefaultMigrator = runnermigrator.NewColumnDefaultMigrator(stores, runnermigrator.EnginesNeedingMigration())
-
// Export archive cleaner
s.exportArchiveCleaner = runnermigrator.NewExportArchiveCleaner(stores)
@@ -245,9 +241,6 @@ func (s *Server) Run(ctx context.Context, port int) error {
s.runnerWG.Add(1)
go s.planCheckScheduler.Run(ctx, &s.runnerWG)
- s.runnerWG.Add(1)
- go s.columnDefaultMigrator.Run(ctx, &s.runnerWG)
-
s.runnerWG.Add(1)
go s.exportArchiveCleaner.Run(ctx, &s.runnerWG)
diff --git a/backend/store/db_schema.go b/backend/store/db_schema.go
index e47639dc9a..3bdfbb7150 100644
--- a/backend/store/db_schema.go
+++ b/backend/store/db_schema.go
@@ -80,7 +80,6 @@ func (s *Store) UpsertDBSchema(
dbMetadata *storepb.DatabaseSchemaMetadata,
dbConfig *storepb.DatabaseConfig,
dbSchema []byte,
- todo bool,
) error {
metadataBytes, err := protojson.Marshal(dbMetadata)
if err != nil {
@@ -97,23 +96,20 @@ func (s *Store) UpsertDBSchema(
db_name,
metadata,
raw_dump,
- config,
- todo
+ config
)
- VALUES (?, ?, ?, ?, ?, ?)
+ VALUES (?, ?, ?, ?, ?)
ON CONFLICT(instance, db_name) DO UPDATE SET
metadata = EXCLUDED.metadata,
raw_dump = EXCLUDED.raw_dump,
- config = EXCLUDED.config,
- todo = EXCLUDED.todo
+ config = EXCLUDED.config
RETURNING metadata, raw_dump, config`,
instanceID,
databaseName,
metadataBytes,
// Convert to string because []byte{} is null which violates db schema constraints.
string(dbSchema),
- configBytes,
- todo)
+ configBytes)
query, args, err := q.ToSQL()
if err != nil {
@@ -183,138 +179,6 @@ func (s *Store) UpdateDBSchema(ctx context.Context, instanceID, databaseName str
return nil
}
-// DBSchemaWithTodo is a struct for a db_schema with the todo column.
-// It is used for the column default value migration.
-type DBSchemaWithTodo struct {
- ID int
- InstanceID string
- DBName string
- Metadata string
-}
-
-// ListDBSchemasWithTodo lists all db_schemas with todo = true.
-func (s *Store) ListDBSchemasWithTodo(ctx context.Context, engineType storepb.Engine, limit int) ([]*DBSchemaWithTodo, error) {
- q := qb.Q().Space(`
- SELECT
- db_schema.id,
- db_schema.instance,
- db_schema.db_name,
- db_schema.metadata
- FROM db_schema
- LEFT JOIN instance ON db_schema.instance = instance.resource_id
- WHERE db_schema.todo = true AND instance.metadata->>'engine' = ?
- LIMIT ?`, engineType.String(), limit)
-
- query, args, err := q.ToSQL()
- if err != nil {
- return nil, errors.Wrapf(err, "failed to build sql")
- }
-
- rows, err := s.GetDB().QueryContext(ctx, query, args...)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
-
- var dbSchemas []*DBSchemaWithTodo
- for rows.Next() {
- var dbSchema DBSchemaWithTodo
- if err := rows.Scan(
- &dbSchema.ID,
- &dbSchema.InstanceID,
- &dbSchema.DBName,
- &dbSchema.Metadata,
- ); err != nil {
- return nil, err
- }
- dbSchemas = append(dbSchemas, &dbSchema)
- }
- if err := rows.Err(); err != nil {
- return nil, err
- }
-
- return dbSchemas, nil
-}
-
-// UpdateDBSchemaTodo updates the todo column of a db_schema.
-func (s *Store) UpdateDBSchemaTodo(ctx context.Context, id int, todo bool) error {
- q := qb.Q().Space("UPDATE db_schema SET todo = ? WHERE id = ?", todo, id)
-
- query, args, err := q.ToSQL()
- if err != nil {
- return errors.Wrapf(err, "failed to build sql")
- }
-
- tx, err := s.GetDB().BeginTx(ctx, nil)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- if _, err := tx.ExecContext(ctx, query, args...); err != nil {
- return err
- }
-
- return tx.Commit()
-}
-
-// UpdateDBSchemaMetadata updates the metadata of a db_schema.
-func (s *Store) UpdateDBSchemaMetadata(ctx context.Context, id int, metadata string) error {
- q := qb.Q().Space("UPDATE db_schema SET metadata = ? WHERE id = ?", metadata, id)
-
- query, args, err := q.ToSQL()
- if err != nil {
- return errors.Wrapf(err, "failed to build sql")
- }
-
- tx, err := s.GetDB().BeginTx(ctx, nil)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- if _, err := tx.ExecContext(ctx, query, args...); err != nil {
- return err
- }
-
- return tx.Commit()
-}
-
-// UpdateDBSchemaMetadataIfTodo updates the metadata and sets todo to false only if todo is currently true.
-// This is used by the migrator to avoid race conditions with the sync process.
-// The WHERE condition ensures atomic check-and-update, preventing any race conditions.
-func (s *Store) UpdateDBSchemaMetadataIfTodo(ctx context.Context, id int, metadata string) error {
- q := qb.Q().Space("UPDATE db_schema SET metadata = ?, todo = false WHERE id = ? AND todo = true", metadata, id)
-
- query, args, err := q.ToSQL()
- if err != nil {
- return errors.Wrapf(err, "failed to build sql")
- }
-
- tx, err := s.GetDB().BeginTx(ctx, nil)
- if err != nil {
- return err
- }
- defer tx.Rollback()
-
- result, err := tx.ExecContext(ctx, query, args...)
- if err != nil {
- return err
- }
-
- rowsAffected, err := result.RowsAffected()
- if err != nil {
- return err
- }
-
- if rowsAffected == 0 {
- // Schema was already processed by sync, skip update
- return tx.Rollback()
- }
-
- return tx.Commit()
-}
-
func (s *Store) convertMetadataAndConfig(ctx context.Context, metadata, schema, config []byte, instanceID string) (*model.DatabaseSchema, error) {
var databaseSchema storepb.DatabaseSchemaMetadata
var databaseConfig storepb.DatabaseConfig