Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: remove magic filename suffix in GitOps workflow (#17643)

Danny Xu5e6e811e540a
diff --git a/action/README.md b/action/README.md
index f3d7aa971a..ff0ccaa1e7 100644
--- a/action/README.md
+++ b/action/README.md
@@ -67,10 +67,18 @@ These flags apply to the main `bytebase-action` command and its subcommands (`ch
- Filenames do not need to follow any versioning format
- Files can be named for clarity and organization (e.g., `tables.sql`, `views.sql`, `indexes.sql`)
- Version is automatically generated from the current timestamp
- - **File Type Detection** (applies to versioned mode only):
- - **DDL (default)**: Standard schema change files (e.g., `v1.0_create_table.sql`)
- - **DML**: Data manipulation files with base filename ending with `dml` (e.g., `v1.0_insert_data_dml.sql`)
- - **DDL Ghost**: Schema changes using gh-ost with base filename ending with `ghost` (e.g., `v1.0_alter_table_ghost.sql`)
+ - **Migration Type Detection** (applies to versioned mode only):
+ - Migration type is specified using a comment at the top of the SQL file
+ - Format: `-- migration-type: ghost`
+ - The comment must appear before any SQL statements
+ - Case insensitive (e.g., `Ghost`, `GHOST`, `ghost` all work)
+ - Only `ghost` is supported for gh-ost migrations
+ - Example:
+ ```sql
+ -- migration-type: ghost
+ ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);
+ ```
+ - If no migration type is specified (or any other value), defaults to `MIGRATION_TYPE_UNSPECIFIED`
- **`--declarative`** (experimental): Use declarative mode for SQL schema management instead of versioned migrations.
- Treats SQL files as desired state definitions rather than incremental changes
diff --git a/action/command/file.go b/action/command/file.go
index 1545a363cb..172be394d6 100644
--- a/action/command/file.go
+++ b/action/command/file.go
@@ -91,17 +91,9 @@ func getReleaseFiles(w *world.World) ([]*v1pb.Release_File, string, error) {
}
base := strings.TrimSuffix(filepath.Base(m), filepath.Ext(m))
- var t v1pb.Release_File_MigrationType
- switch {
- case strings.HasSuffix(base, "dml"):
- t = v1pb.Release_File_DML
- case strings.HasSuffix(base, "ghost"):
- t = v1pb.Release_File_DDL_GHOST
- case strings.HasSuffix(base, "ddl"):
- t = v1pb.Release_File_DDL
- default:
- t = v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED
- }
+
+ // Extract migration type from SQL front matter comments
+ t := extractMigrationTypeFromContent(string(content))
version := extractVersion(base)
if version == "" {
@@ -139,3 +131,29 @@ func extractVersion(s string) string {
// Return the first capture group which contains just the version numbers
return matches[1]
}
+
+var migrationTypeReg = regexp.MustCompile(`(?i)^--\s*migration-type:\s*(\w+)`)
+
+// extractMigrationTypeFromContent extracts migration type from SQL front matter comments.
+// Supported values: ghost
+// Example:
+//
+// -- migration-type: ghost
+// ALTER TABLE ...
+func extractMigrationTypeFromContent(content string) v1pb.Release_File_MigrationType {
+ lines := strings.Split(content, "\n")
+ for _, line := range lines {
+ line = strings.TrimSpace(line)
+ // Stop at first non-comment line
+ if line != "" && !strings.HasPrefix(line, "--") {
+ break
+ }
+ matches := migrationTypeReg.FindStringSubmatch(line)
+ if len(matches) >= 2 {
+ if strings.ToLower(matches[1]) == "ghost" {
+ return v1pb.Release_File_DDL_GHOST
+ }
+ }
+ }
+ return v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED
+}
diff --git a/action/command/file_test.go b/action/command/file_test.go
index c235ff66ac..fc1c1bd5f0 100644
--- a/action/command/file_test.go
+++ b/action/command/file_test.go
@@ -4,6 +4,8 @@ import (
"testing"
"github.com/stretchr/testify/require"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
)
func TestExtractVersion(t *testing.T) {
@@ -96,3 +98,101 @@ func TestExtractVersion(t *testing.T) {
})
}
}
+
+func TestExtractMigrationTypeFromContent(t *testing.T) {
+ tests := []struct {
+ name string
+ content string
+ expected v1pb.Release_File_MigrationType
+ }{
+ {
+ name: "ghost type",
+ content: `-- migration-type: ghost
+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,
+ expected: v1pb.Release_File_DDL_GHOST,
+ },
+ {
+ name: "case insensitive Ghost",
+ content: `-- migration-type: Ghost
+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,
+ expected: v1pb.Release_File_DDL_GHOST,
+ },
+ {
+ name: "case insensitive GHOST",
+ content: `-- migration-type: GHOST
+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,
+ expected: v1pb.Release_File_DDL_GHOST,
+ },
+ {
+ name: "with extra spaces",
+ content: `-- migration-type: ghost
+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,
+ expected: v1pb.Release_File_DDL_GHOST,
+ },
+ {
+ name: "with multiple comment lines",
+ content: `-- This is a migration file
+-- Author: John Doe
+-- migration-type: ghost
+-- Date: 2024-01-01
+
+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,
+ expected: v1pb.Release_File_DDL_GHOST,
+ },
+ {
+ name: "no migration type specified",
+ content: `-- This is a migration file
+ALTER TABLE users ADD COLUMN age INT;`,
+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,
+ },
+ {
+ name: "ddl type - should be unspecified",
+ content: `-- migration-type: ddl
+ALTER TABLE users ADD COLUMN age INT;`,
+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,
+ },
+ {
+ name: "dml type - should be unspecified",
+ content: `-- migration-type: dml
+UPDATE users SET active = true;`,
+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,
+ },
+ {
+ name: "migration type after SQL statement",
+ content: `ALTER TABLE users ADD COLUMN age INT;
+-- migration-type: ghost`,
+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,
+ },
+ {
+ name: "invalid migration type",
+ content: `-- migration-type: invalid
+ALTER TABLE users ADD COLUMN age INT;`,
+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,
+ },
+ {
+ name: "empty content",
+ content: ``,
+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,
+ },
+ {
+ name: "only comments with ghost",
+ content: `-- migration-type: ghost
+-- More comments`,
+ expected: v1pb.Release_File_DDL_GHOST,
+ },
+ {
+ name: "with blank lines before statement",
+ content: `-- migration-type: ghost
+
+ALTER TABLE users ADD COLUMN age INT;`,
+ expected: v1pb.Release_File_DDL_GHOST,
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ result := extractMigrationTypeFromContent(tt.content)
+ require.Equal(t, tt.expected, result)
+ })
+ }
+}
diff --git a/backend/api/v1/plan_service_plan_check.go b/backend/api/v1/plan_service_plan_check.go
index d9a60be5e8..3ffa96ca1e 100644
--- a/backend/api/v1/plan_service_plan_check.go
+++ b/backend/api/v1/plan_service_plan_check.go
@@ -88,10 +88,11 @@ func getPlanCheckRunsFromSpec(ctx context.Context, s *store.Store, plan *store.P
func getPlanCheckRunsFromChangeDatabaseConfigDatabaseGroupTarget(ctx context.Context, s *store.Store, plan *store.PlanMessage, config *storepb.PlanConfig_ChangeDatabaseConfig) ([]*store.PlanCheckRunMessage, error) {
switch config.Type {
case storepb.PlanConfig_ChangeDatabaseConfig_MIGRATE:
- // Only DDL and DML migrate types are supported for database group targets
+ // Only DDL, DML, and MIGRATE_TYPE_UNSPECIFIED (treated as DDL) migrate types are supported for database group targets
switch config.MigrateType {
case storepb.PlanConfig_ChangeDatabaseConfig_DDL:
case storepb.PlanConfig_ChangeDatabaseConfig_DML:
+ case storepb.PlanConfig_ChangeDatabaseConfig_MIGRATE_TYPE_UNSPECIFIED:
default:
return nil, errors.Errorf("unsupported migrate type %q for database group target", config.MigrateType)
}
diff --git a/backend/plugin/advisor/change_type.go b/backend/plugin/advisor/change_type.go
index a69cd2929f..e33f2c3ef0 100644
--- a/backend/plugin/advisor/change_type.go
+++ b/backend/plugin/advisor/change_type.go
@@ -7,20 +7,8 @@ var sqlEditorAllowlist = map[SQLReviewRuleType]bool{
}
func isRuleAllowed(rule SQLReviewRuleType, changeType storepb.PlanCheckRunConfig_ChangeDatabaseType) bool {
- switch changeType {
- case storepb.PlanCheckRunConfig_CHANGE_DATABASE_TYPE_UNSPECIFIED:
- return false
- case storepb.PlanCheckRunConfig_DDL:
- return true
- case storepb.PlanCheckRunConfig_DDL_GHOST:
- return true
- case storepb.PlanCheckRunConfig_DML:
- return true
- case storepb.PlanCheckRunConfig_SDL:
- return true
- case storepb.PlanCheckRunConfig_SQL_EDITOR:
+ if changeType == storepb.PlanCheckRunConfig_SQL_EDITOR {
return sqlEditorAllowlist[rule]
- default:
- return false
}
+ return true
}
diff --git a/backend/runner/plancheck/statement_advise_executor.go b/backend/runner/plancheck/statement_advise_executor.go
index bc47451b8a..05d67d33b1 100644
--- a/backend/runner/plancheck/statement_advise_executor.go
+++ b/backend/runner/plancheck/statement_advise_executor.go
@@ -46,9 +46,7 @@ type StatementAdviseExecutor struct {
// Run will run the plan check statement advise executor once, and run its sub-advisors one-by-one.
func (e *StatementAdviseExecutor) Run(ctx context.Context, config *storepb.PlanCheckRunConfig) ([]*storepb.PlanCheckRunResult_Result, error) {
- if config.ChangeDatabaseType == storepb.PlanCheckRunConfig_CHANGE_DATABASE_TYPE_UNSPECIFIED {
- return nil, errors.Errorf("change database type is unspecified")
- }
+ changeType := config.ChangeDatabaseType
sheetUID := int(config.SheetUid)
sheet, err := e.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID})
@@ -72,7 +70,6 @@ func (e *StatementAdviseExecutor) Run(ctx context.Context, config *storepb.PlanC
if err != nil {
return nil, err
}
- changeType := config.ChangeDatabaseType
enablePriorBackup := config.EnablePriorBackup
instance, err := e.store.GetInstanceV2(ctx, &store.FindInstanceMessage{ResourceID: &config.InstanceId})
diff --git a/backend/runner/taskrun/executor.go b/backend/runner/taskrun/executor.go
index d93ac26c00..c80ca1c73a 100644
--- a/backend/runner/taskrun/executor.go
+++ b/backend/runner/taskrun/executor.go
@@ -653,8 +653,7 @@ func convertTaskType(t *store.TaskMessage) (storepb.ChangelogPayload_Type, store
func isChangeDatabaseTask(task *store.TaskMessage) bool {
switch task.Type {
case storepb.Task_DATABASE_MIGRATE:
- // All DATABASE_MIGRATE tasks involve changing a database except possibly UNSPECIFIED
- return task.Payload.GetMigrateType() != storepb.Task_MIGRATE_TYPE_UNSPECIFIED
+ return true
case storepb.Task_DATABASE_SDL:
return true
case storepb.Task_DATABASE_CREATE,
diff --git a/backend/runner/taskrun/schedulerv2.go b/backend/runner/taskrun/schedulerv2.go
index 2b4d62f1cb..e9cdcdf6ee 100644
--- a/backend/runner/taskrun/schedulerv2.go
+++ b/backend/runner/taskrun/schedulerv2.go
@@ -1002,14 +1002,14 @@ func isSequentialTask(task *store.TaskMessage) bool {
//exhaustive:enforce
switch task.Type {
case storepb.Task_DATABASE_MIGRATE:
- // DDL and GHOST operations should be sequential
+ // DDL, GHOST, and MIGRATE_TYPE_UNSPECIFIED (treated as DDL) operations should be sequential
switch task.Payload.GetMigrateType() {
- case storepb.Task_DDL, storepb.Task_GHOST:
+ case storepb.Task_DDL, storepb.Task_GHOST, storepb.Task_MIGRATE_TYPE_UNSPECIFIED:
return true
case storepb.Task_DML:
return false
default:
- return false
+ return true
}
case storepb.Task_DATABASE_SDL:
// SDL operations should be sequential