Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
chore: add pre-flight validation to gh-ost workflow (#17639)
Vincent Huangd7a8c4f62c6e
diff --git a/backend/component/ghost/config.go b/backend/component/ghost/config.goindex bd59dc2d67..1019ea9b64 100644--- a/backend/component/ghost/config.go+++ b/backend/component/ghost/config.go@@ -2,6 +2,8 @@ package ghostimport ("context"+ "log/slog"+ "os""strconv""strings"@@ -280,6 +282,21 @@ func NewMigrationContext(ctx context.Context, taskID int, database *store.DatabamigrationContext.SetChunkSize(defaultConfig.chunkSize)migrationContext.SetDMLBatchSize(defaultConfig.dmlBatchSize)migrationContext.SetMaxLagMillisecondsThrottleThreshold(defaultConfig.maxLagMillisecondsThrottleThreshold)+ // Check for environment variable to limit auth retries+ if maxAuthRetries := os.Getenv("GHOST_MAX_AUTH_RETRIES"); maxAuthRetries != "" {+ if retries, err := strconv.Atoi(maxAuthRetries); err == nil && retries > 0 {+ migrationContext.MaxAuthFailures = retries+ slog.Info("gh-ost auth retry limit set",+ slog.Int("max_failures", migrationContext.MaxAuthFailures),+ slog.String("source", "environment"))+ }+ } else {+ // Default to 10 retries to prevent retry storms+ migrationContext.MaxAuthFailures = 10+ slog.Info("gh-ost auth retry limit set",+ slog.Int("max_failures", migrationContext.MaxAuthFailures),+ slog.String("source", "default"))+ }migrationContext.SetDefaultNumRetries(defaultConfig.defaultNumRetries)migrationContext.ApplyCredentials()if err := migrationContext.SetCutOverLockTimeoutSeconds(defaultConfig.cutoverLockTimeoutSeconds); err != nil {diff --git a/backend/component/ghost/validator.go b/backend/component/ghost/validator.gonew file mode 100644index 0000000000..8c3af2a0bd--- /dev/null+++ b/backend/component/ghost/validator.go@@ -0,0 +1,177 @@+package ghost++import (+ "context"+ "fmt"+ "log/slog"+ "strings"++ "github.com/pkg/errors"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+)++// BinlogValidationResult contains detailed results of binlog access validation+type BinlogValidationResult struct {+ // Core validation state+ Valid bool+ Error error++ // Detailed findings for specific error messages+ BinlogEnabled bool+ BinlogFormat string+ HasPrivilege bool+ MissingPrivileges []string // Specific privileges that are missing+ CurrentGrants []string // Current grants for debugging+}++// ValidateBinlogAccess performs comprehensive validation of gh-ost prerequisites+// Returns a structured result that can be used for both plan checks and execution+func ValidateBinlogAccess(ctx context.Context, driver db.Driver, adminDataSource *storepb.DataSource) *BinlogValidationResult {+ result := &BinlogValidationResult{+ Valid: true,+ BinlogEnabled: false,+ HasPrivilege: false,+ MissingPrivileges: []string{},+ CurrentGrants: []string{},+ }++ // Test 1: Check if we can access binary log status+ // Try both old and new MySQL commands for compatibility+ canAccessBinlog := false+ if _, err := driver.GetDB().ExecContext(ctx, "SHOW MASTER STATUS"); err == nil {+ canAccessBinlog = true+ } else if _, err := driver.GetDB().ExecContext(ctx, "SHOW BINARY LOG STATUS"); err == nil {+ canAccessBinlog = true+ }++ if !canAccessBinlog {+ result.Valid = false+ result.Error = errors.New("cannot access binary logs - ensure user has REPLICATION CLIENT privilege")+ slog.Error("binlog access validation failed: cannot access binary logs",+ slog.String("host", adminDataSource.GetHost()),+ slog.String("user", adminDataSource.GetUsername()))+ return result+ }++ // Test 2: Check if binary logging is enabled+ var logBin string+ row := driver.GetDB().QueryRowContext(ctx, "SELECT @@log_bin")+ if err := row.Scan(&logBin); err != nil {+ result.Valid = false+ result.Error = errors.Wrap(err, "failed to check if binary logging is enabled")+ return result+ }++ result.BinlogEnabled = (logBin == "1" || strings.ToUpper(logBin) == "ON")+ if !result.BinlogEnabled {+ result.Valid = false+ result.Error = errors.New("binary logging is not enabled on this MySQL instance")+ return result+ }++ // Test 3: Check user privileges+ rows, err := driver.GetDB().QueryContext(ctx, "SHOW GRANTS")+ if err != nil {+ result.Valid = false+ result.Error = errors.Wrap(err, "failed to check user grants")+ return result+ }+ defer rows.Close()++ for rows.Next() {+ var grant string+ if err := rows.Scan(&grant); err != nil {+ slog.Warn("failed to scan grant",+ slog.String("host", adminDataSource.GetHost()),+ slog.String("user", adminDataSource.GetUsername()),+ slog.String("error", err.Error()))+ continue+ }+ result.CurrentGrants = append(result.CurrentGrants, grant)++ upperGrant := strings.ToUpper(grant)+ if strings.Contains(upperGrant, "REPLICATION SLAVE") ||+ strings.Contains(upperGrant, "ALL PRIVILEGES") {+ result.HasPrivilege = true+ }+ }++ if err := rows.Err(); err != nil {+ result.Valid = false+ result.Error = errors.Wrap(err, "error reading grants")+ return result+ }++ if !result.HasPrivilege {+ result.Valid = false+ result.MissingPrivileges = append(result.MissingPrivileges, "REPLICATION SLAVE")+ result.Error = errors.New("user does not have REPLICATION SLAVE privilege required for gh-ost")+ slog.Error("missing REPLICATION SLAVE privilege",+ slog.String("host", adminDataSource.GetHost()),+ slog.String("user", adminDataSource.GetUsername()),+ slog.Any("grants", result.CurrentGrants))+ return result+ }++ // Test 4: Check binlog format (must be ROW or MIXED, not STATEMENT)+ row = driver.GetDB().QueryRowContext(ctx, "SELECT @@binlog_format")+ if err := row.Scan(&result.BinlogFormat); err != nil {+ result.Valid = false+ result.Error = errors.Wrap(err, "failed to check binlog format")+ return result+ }++ if strings.ToUpper(result.BinlogFormat) == "STATEMENT" {+ result.Valid = false+ result.Error = errors.Errorf("binlog_format is %s, but gh-ost requires ROW or MIXED format", result.BinlogFormat)+ return result+ }++ // All checks passed+ slog.Info("binlog access validation passed",+ slog.String("host", adminDataSource.GetHost()),+ slog.String("user", adminDataSource.GetUsername()),+ slog.String("binlog_format", result.BinlogFormat))++ return result+}++// GetUserFriendlyError returns a user-friendly error message based on validation results+func (r *BinlogValidationResult) GetUserFriendlyError() (title, content string) {+ if r.Valid {+ return "", ""+ }++ title = "gh-ost migration prerequisites not met"++ if !r.BinlogEnabled {+ content = "Binary logging is not enabled on this MySQL instance. Please enable it with:\n" ++ "SET GLOBAL log_bin=ON (requires MySQL restart)"+ return title, content+ }++ if !r.HasPrivilege && len(r.MissingPrivileges) > 0 {+ content = fmt.Sprintf("Database user is missing required privilege: %s\n", strings.Join(r.MissingPrivileges, ", ")) ++ "Please grant it with:\n" ++ "GRANT REPLICATION SLAVE ON *.* TO 'user'@'host'"+ return title, content+ }++ if r.BinlogFormat == "STATEMENT" {+ content = fmt.Sprintf("Current binlog_format is %s, but gh-ost requires ROW or MIXED format.\n", r.BinlogFormat) ++ "Please change it with:\n" ++ "SET GLOBAL binlog_format='ROW'"+ return title, content+ }++ // Generic error fallback+ if r.Error != nil {+ content = fmt.Sprintf("Validation failed: %v", r.Error)+ } else {+ content = "Unknown validation error occurred"+ }++ return title, content+}diff --git a/backend/runner/plancheck/ghost_sync_executor.go b/backend/runner/plancheck/ghost_sync_executor.goindex 6d204bbdce..cbd8a86ebe 100644--- a/backend/runner/plancheck/ghost_sync_executor.go+++ b/backend/runner/plancheck/ghost_sync_executor.go@@ -107,6 +107,34 @@ func (e *GhostSyncExecutor) Run(ctx context.Context, config *storepb.PlanCheckRureturn nil, common.Wrapf(err, common.Internal, "failed to parse table name from statement, statement: %v", statement)}+ // Validate binlog access before attempting migration+ // This prevents retry storms and provides early feedback in plan checks+ driver, err := e.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{})+ if err != nil {+ return []*storepb.PlanCheckRunResult_Result{+ {+ Status: storepb.PlanCheckRunResult_Result_ERROR,+ Title: "Failed to connect to database",+ Content: fmt.Sprintf("Cannot establish connection: %v", err),+ Code: common.Internal.Int32(),+ },+ }, nil+ }+ defer driver.Close(ctx)++ validationResult := ghost.ValidateBinlogAccess(ctx, driver, adminDataSource)+ if !validationResult.Valid {+ title, content := validationResult.GetUserFriendlyError()+ return []*storepb.PlanCheckRunResult_Result{+ {+ Status: storepb.PlanCheckRunResult_Result_ERROR,+ Title: title,+ Content: content,+ Code: common.Internal.Int32(),+ },+ }, nil+ }+migrationContext, err := ghost.NewMigrationContext(ctx, rand.Intn(10000000), database, adminDataSource, tableName, fmt.Sprintf("_dryrun_%d", time.Now().Unix()), statement, true, config.GhostFlags, 20000000)if err != nil {return nil, common.Wrapf(err, common.Internal, "failed to create migration context")@@ -122,12 +150,14 @@ func (e *GhostSyncExecutor) Run(ctx context.Context, config *storepb.PlanCheckRudefer func() {if err := func() error {- ctx := context.Background()- driver, err := e.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{})+ // Note: We're reusing the ctx from parent scope and creating a new driver+ // because the original driver was already closed+ cleanupCtx := context.Background()+ cleanupDriver, err := e.dbFactory.GetAdminDatabaseDriver(cleanupCtx, instance, database, db.ConnectionContext{})if err != nil {- return errors.Wrapf(err, "failed to get driver")+ return errors.Wrapf(err, "failed to get driver for cleanup")}- defer driver.Close(ctx)+ defer cleanupDriver.Close(cleanupCtx)// Use the backup database name of MySQL as the ghost database name.ghostDBName := common.BackupDatabaseNameOfEngine(storepb.Engine_MYSQL)@@ -138,7 +168,7 @@ func (e *GhostSyncExecutor) Run(ctx context.Context, config *storepb.PlanCheckRumigrationContext.GetChangelogTableName(),)- if _, err := driver.GetDB().ExecContext(ctx, sql); err != nil {+ if _, err := cleanupDriver.GetDB().ExecContext(cleanupCtx, sql); err != nil {return errors.Wrapf(err, "failed to drop gh-ost temp tables")}return nildiff --git a/go.mod b/go.modindex 12bd204a89..cebe41f0b6 100644--- a/go.mod+++ b/go.mod@@ -402,7 +402,7 @@ replace (// fix potential security issue(CVE-2020-26160) introduced by indirect dependency.github.com/dgrijalva/jwt-go => github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible// Other fixes.- github.com/github/gh-ost => github.com/bytebase/gh-ost2 v1.1.7-0.20250310031106-33e16ca4b2e0+ github.com/github/gh-ost => github.com/bytebase/gh-ost2 v1.1.7-0.20251002210738-35e5dddaad7cgithub.com/jackc/pgx/v5 => github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767diff --git a/go.sum b/go.sumindex 460d0c6eb1..1b01353346 100644--- a/go.sum+++ b/go.sum@@ -809,8 +809,8 @@ github.com/bytebase/cosmosdb-parser v0.0.0-20250317064006-c1dcb3ed4fa4 h1:mw6ze5github.com/bytebase/cosmosdb-parser v0.0.0-20250317064006-c1dcb3ed4fa4/go.mod h1:sivaOoKlMrXkuude2VGsKv6p83CRRGgblNfeVKQka/Y=github.com/bytebase/doris-parser v0.0.0-20250512080703-bacdebf2f435 h1:Kr/zjp7arILvloH8IPnBYz8K7ntJxhRwLFnUa2m2z0E=github.com/bytebase/doris-parser v0.0.0-20250512080703-bacdebf2f435/go.mod h1:mfKfCbkm5QFVKqZIwRxlUaY26kR3ttYeCiDLV4aUg7k=-github.com/bytebase/gh-ost2 v1.1.7-0.20250310031106-33e16ca4b2e0 h1:wm4Set2lwArmC4f9hYdiunXtEYYzr6SyYRepTXNdkm4=-github.com/bytebase/gh-ost2 v1.1.7-0.20250310031106-33e16ca4b2e0/go.mod h1:X0CGYwIZUnjfrv02/nBlftxqSBMm4mYJzFwiyn2D4d8=+github.com/bytebase/gh-ost2 v1.1.7-0.20251002210738-35e5dddaad7c h1:scJ4ycKuUAaA5XMMq9/SQdWuupG6flnlq7v3Z65X0Dk=+github.com/bytebase/gh-ost2 v1.1.7-0.20251002210738-35e5dddaad7c/go.mod h1:X0CGYwIZUnjfrv02/nBlftxqSBMm4mYJzFwiyn2D4d8=github.com/bytebase/go-mssqldb v0.0.0-20240801091126-3ff3ca07d898 h1:0ggqA9XL4yvtePPvRDRYMed3jtdNk4s0o8AMW8uJPSA=github.com/bytebase/go-mssqldb v0.0.0-20240801091126-3ff3ca07d898/go.mod h1:2KhUz4GSLcdJGXSJMXAf2BjmakcgoOkNyPpHdGAxRpY=github.com/bytebase/gohive v0.0.0-20240422092929-d76993a958a4 h1:5o+GWOu+j68eAODMw7/lFJavWCCiPMeqEHuOGx0YmoU=