Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: remove magic filename suffix in GitOps workflow (#17643)
Danny Xu5e6e811e540a
diff --git a/action/README.md b/action/README.mdindex f3d7aa971a..ff0ccaa1e7 100644--- a/action/README.md+++ b/action/README.md@@ -67,10 +67,18 @@ These flags apply to the main `bytebase-action` command and its subcommands (`ch- Filenames do not need to follow any versioning format- Files can be named for clarity and organization (e.g., `tables.sql`, `views.sql`, `indexes.sql`)- Version is automatically generated from the current timestamp- - **File Type Detection** (applies to versioned mode only):- - **DDL (default)**: Standard schema change files (e.g., `v1.0_create_table.sql`)- - **DML**: Data manipulation files with base filename ending with `dml` (e.g., `v1.0_insert_data_dml.sql`)- - **DDL Ghost**: Schema changes using gh-ost with base filename ending with `ghost` (e.g., `v1.0_alter_table_ghost.sql`)+ - **Migration Type Detection** (applies to versioned mode only):+ - Migration type is specified using a comment at the top of the SQL file+ - Format: `-- migration-type: ghost`+ - The comment must appear before any SQL statements+ - Case insensitive (e.g., `Ghost`, `GHOST`, `ghost` all work)+ - Only `ghost` is supported for gh-ost migrations+ - Example:+ ```sql+ -- migration-type: ghost+ ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);+ ```+ - If no migration type is specified (or any other value), defaults to `MIGRATION_TYPE_UNSPECIFIED`- **`--declarative`** (experimental): Use declarative mode for SQL schema management instead of versioned migrations.- Treats SQL files as desired state definitions rather than incremental changesdiff --git a/action/command/file.go b/action/command/file.goindex 1545a363cb..172be394d6 100644--- a/action/command/file.go+++ b/action/command/file.go@@ -91,17 +91,9 @@ func getReleaseFiles(w *world.World) ([]*v1pb.Release_File, string, error) {}base := strings.TrimSuffix(filepath.Base(m), filepath.Ext(m))- var t v1pb.Release_File_MigrationType- switch {- case strings.HasSuffix(base, "dml"):- t = v1pb.Release_File_DML- case strings.HasSuffix(base, "ghost"):- t = v1pb.Release_File_DDL_GHOST- case strings.HasSuffix(base, "ddl"):- t = v1pb.Release_File_DDL- default:- t = v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED- }++ // Extract migration type from SQL front matter comments+ t := extractMigrationTypeFromContent(string(content))version := extractVersion(base)if version == "" {@@ -139,3 +131,29 @@ func extractVersion(s string) string {// Return the first capture group which contains just the version numbersreturn matches[1]}++var migrationTypeReg = regexp.MustCompile(`(?i)^--\s*migration-type:\s*(\w+)`)++// extractMigrationTypeFromContent extracts migration type from SQL front matter comments.+// Supported values: ghost+// Example:+//+// -- migration-type: ghost+// ALTER TABLE ...+func extractMigrationTypeFromContent(content string) v1pb.Release_File_MigrationType {+ lines := strings.Split(content, "\n")+ for _, line := range lines {+ line = strings.TrimSpace(line)+ // Stop at first non-comment line+ if line != "" && !strings.HasPrefix(line, "--") {+ break+ }+ matches := migrationTypeReg.FindStringSubmatch(line)+ if len(matches) >= 2 {+ if strings.ToLower(matches[1]) == "ghost" {+ return v1pb.Release_File_DDL_GHOST+ }+ }+ }+ return v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED+}diff --git a/action/command/file_test.go b/action/command/file_test.goindex c235ff66ac..fc1c1bd5f0 100644--- a/action/command/file_test.go+++ b/action/command/file_test.go@@ -4,6 +4,8 @@ import ("testing""github.com/stretchr/testify/require"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1")func TestExtractVersion(t *testing.T) {@@ -96,3 +98,101 @@ func TestExtractVersion(t *testing.T) {})}}++func TestExtractMigrationTypeFromContent(t *testing.T) {+ tests := []struct {+ name string+ content string+ expected v1pb.Release_File_MigrationType+ }{+ {+ name: "ghost type",+ content: `-- migration-type: ghost+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,+ expected: v1pb.Release_File_DDL_GHOST,+ },+ {+ name: "case insensitive Ghost",+ content: `-- migration-type: Ghost+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,+ expected: v1pb.Release_File_DDL_GHOST,+ },+ {+ name: "case insensitive GHOST",+ content: `-- migration-type: GHOST+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,+ expected: v1pb.Release_File_DDL_GHOST,+ },+ {+ name: "with extra spaces",+ content: `-- migration-type: ghost+ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,+ expected: v1pb.Release_File_DDL_GHOST,+ },+ {+ name: "with multiple comment lines",+ content: `-- This is a migration file+-- Author: John Doe+-- migration-type: ghost+-- Date: 2024-01-01++ALTER TABLE large_table ADD COLUMN new_col VARCHAR(255);`,+ expected: v1pb.Release_File_DDL_GHOST,+ },+ {+ name: "no migration type specified",+ content: `-- This is a migration file+ALTER TABLE users ADD COLUMN age INT;`,+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,+ },+ {+ name: "ddl type - should be unspecified",+ content: `-- migration-type: ddl+ALTER TABLE users ADD COLUMN age INT;`,+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,+ },+ {+ name: "dml type - should be unspecified",+ content: `-- migration-type: dml+UPDATE users SET active = true;`,+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,+ },+ {+ name: "migration type after SQL statement",+ content: `ALTER TABLE users ADD COLUMN age INT;+-- migration-type: ghost`,+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,+ },+ {+ name: "invalid migration type",+ content: `-- migration-type: invalid+ALTER TABLE users ADD COLUMN age INT;`,+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,+ },+ {+ name: "empty content",+ content: ``,+ expected: v1pb.Release_File_MIGRATION_TYPE_UNSPECIFIED,+ },+ {+ name: "only comments with ghost",+ content: `-- migration-type: ghost+-- More comments`,+ expected: v1pb.Release_File_DDL_GHOST,+ },+ {+ name: "with blank lines before statement",+ content: `-- migration-type: ghost++ALTER TABLE users ADD COLUMN age INT;`,+ expected: v1pb.Release_File_DDL_GHOST,+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ result := extractMigrationTypeFromContent(tt.content)+ require.Equal(t, tt.expected, result)+ })+ }+}diff --git a/backend/api/v1/plan_service_plan_check.go b/backend/api/v1/plan_service_plan_check.goindex d9a60be5e8..3ffa96ca1e 100644--- a/backend/api/v1/plan_service_plan_check.go+++ b/backend/api/v1/plan_service_plan_check.go@@ -88,10 +88,11 @@ func getPlanCheckRunsFromSpec(ctx context.Context, s *store.Store, plan *store.Pfunc getPlanCheckRunsFromChangeDatabaseConfigDatabaseGroupTarget(ctx context.Context, s *store.Store, plan *store.PlanMessage, config *storepb.PlanConfig_ChangeDatabaseConfig) ([]*store.PlanCheckRunMessage, error) {switch config.Type {case storepb.PlanConfig_ChangeDatabaseConfig_MIGRATE:- // Only DDL and DML migrate types are supported for database group targets+ // Only DDL, DML, and MIGRATE_TYPE_UNSPECIFIED (treated as DDL) migrate types are supported for database group targetsswitch config.MigrateType {case storepb.PlanConfig_ChangeDatabaseConfig_DDL:case storepb.PlanConfig_ChangeDatabaseConfig_DML:+ case storepb.PlanConfig_ChangeDatabaseConfig_MIGRATE_TYPE_UNSPECIFIED:default:return nil, errors.Errorf("unsupported migrate type %q for database group target", config.MigrateType)}diff --git a/backend/plugin/advisor/change_type.go b/backend/plugin/advisor/change_type.goindex a69cd2929f..e33f2c3ef0 100644--- a/backend/plugin/advisor/change_type.go+++ b/backend/plugin/advisor/change_type.go@@ -7,20 +7,8 @@ var sqlEditorAllowlist = map[SQLReviewRuleType]bool{}func isRuleAllowed(rule SQLReviewRuleType, changeType storepb.PlanCheckRunConfig_ChangeDatabaseType) bool {- switch changeType {- case storepb.PlanCheckRunConfig_CHANGE_DATABASE_TYPE_UNSPECIFIED:- return false- case storepb.PlanCheckRunConfig_DDL:- return true- case storepb.PlanCheckRunConfig_DDL_GHOST:- return true- case storepb.PlanCheckRunConfig_DML:- return true- case storepb.PlanCheckRunConfig_SDL:- return true- case storepb.PlanCheckRunConfig_SQL_EDITOR:+ if changeType == storepb.PlanCheckRunConfig_SQL_EDITOR {return sqlEditorAllowlist[rule]- default:- return false}+ return true}diff --git a/backend/runner/plancheck/statement_advise_executor.go b/backend/runner/plancheck/statement_advise_executor.goindex bc47451b8a..05d67d33b1 100644--- a/backend/runner/plancheck/statement_advise_executor.go+++ b/backend/runner/plancheck/statement_advise_executor.go@@ -46,9 +46,7 @@ type StatementAdviseExecutor struct {// Run will run the plan check statement advise executor once, and run its sub-advisors one-by-one.func (e *StatementAdviseExecutor) Run(ctx context.Context, config *storepb.PlanCheckRunConfig) ([]*storepb.PlanCheckRunResult_Result, error) {- if config.ChangeDatabaseType == storepb.PlanCheckRunConfig_CHANGE_DATABASE_TYPE_UNSPECIFIED {- return nil, errors.Errorf("change database type is unspecified")- }+ changeType := config.ChangeDatabaseTypesheetUID := int(config.SheetUid)sheet, err := e.store.GetSheet(ctx, &store.FindSheetMessage{UID: &sheetUID})@@ -72,7 +70,6 @@ func (e *StatementAdviseExecutor) Run(ctx context.Context, config *storepb.PlanCif err != nil {return nil, err}- changeType := config.ChangeDatabaseTypeenablePriorBackup := config.EnablePriorBackupinstance, err := e.store.GetInstanceV2(ctx, &store.FindInstanceMessage{ResourceID: &config.InstanceId})diff --git a/backend/runner/taskrun/executor.go b/backend/runner/taskrun/executor.goindex d93ac26c00..c80ca1c73a 100644--- a/backend/runner/taskrun/executor.go+++ b/backend/runner/taskrun/executor.go@@ -653,8 +653,7 @@ func convertTaskType(t *store.TaskMessage) (storepb.ChangelogPayload_Type, storefunc isChangeDatabaseTask(task *store.TaskMessage) bool {switch task.Type {case storepb.Task_DATABASE_MIGRATE:- // All DATABASE_MIGRATE tasks involve changing a database except possibly UNSPECIFIED- return task.Payload.GetMigrateType() != storepb.Task_MIGRATE_TYPE_UNSPECIFIED+ return truecase storepb.Task_DATABASE_SDL:return truecase storepb.Task_DATABASE_CREATE,diff --git a/backend/runner/taskrun/schedulerv2.go b/backend/runner/taskrun/schedulerv2.goindex 2b4d62f1cb..e9cdcdf6ee 100644--- a/backend/runner/taskrun/schedulerv2.go+++ b/backend/runner/taskrun/schedulerv2.go@@ -1002,14 +1002,14 @@ func isSequentialTask(task *store.TaskMessage) bool {//exhaustive:enforceswitch task.Type {case storepb.Task_DATABASE_MIGRATE:- // DDL and GHOST operations should be sequential+ // DDL, GHOST, and MIGRATE_TYPE_UNSPECIFIED (treated as DDL) operations should be sequentialswitch task.Payload.GetMigrateType() {- case storepb.Task_DDL, storepb.Task_GHOST:+ case storepb.Task_DDL, storepb.Task_GHOST, storepb.Task_MIGRATE_TYPE_UNSPECIFIED:return truecase storepb.Task_DML:return falsedefault:- return false+ return true}case storepb.Task_DATABASE_SDL:// SDL operations should be sequential