Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

chore: add pre-flight validation to gh-ost workflow (#17639)

Vincent Huangd7a8c4f62c6e
diff --git a/backend/component/ghost/config.go b/backend/component/ghost/config.go
index bd59dc2d67..1019ea9b64 100644
--- a/backend/component/ghost/config.go
+++ b/backend/component/ghost/config.go
@@ -2,6 +2,8 @@ package ghost
import (
"context"
+ "log/slog"
+ "os"
"strconv"
"strings"
@@ -280,6 +282,21 @@ func NewMigrationContext(ctx context.Context, taskID int, database *store.Databa
migrationContext.SetChunkSize(defaultConfig.chunkSize)
migrationContext.SetDMLBatchSize(defaultConfig.dmlBatchSize)
migrationContext.SetMaxLagMillisecondsThrottleThreshold(defaultConfig.maxLagMillisecondsThrottleThreshold)
+ // Check for environment variable to limit auth retries
+ if maxAuthRetries := os.Getenv("GHOST_MAX_AUTH_RETRIES"); maxAuthRetries != "" {
+ if retries, err := strconv.Atoi(maxAuthRetries); err == nil && retries > 0 {
+ migrationContext.MaxAuthFailures = retries
+ slog.Info("gh-ost auth retry limit set",
+ slog.Int("max_failures", migrationContext.MaxAuthFailures),
+ slog.String("source", "environment"))
+ }
+ } else {
+ // Default to 10 retries to prevent retry storms
+ migrationContext.MaxAuthFailures = 10
+ slog.Info("gh-ost auth retry limit set",
+ slog.Int("max_failures", migrationContext.MaxAuthFailures),
+ slog.String("source", "default"))
+ }
migrationContext.SetDefaultNumRetries(defaultConfig.defaultNumRetries)
migrationContext.ApplyCredentials()
if err := migrationContext.SetCutOverLockTimeoutSeconds(defaultConfig.cutoverLockTimeoutSeconds); err != nil {
diff --git a/backend/component/ghost/validator.go b/backend/component/ghost/validator.go
new file mode 100644
index 0000000000..8c3af2a0bd
--- /dev/null
+++ b/backend/component/ghost/validator.go
@@ -0,0 +1,177 @@
+package ghost
+
+import (
+ "context"
+ "fmt"
+ "log/slog"
+ "strings"
+
+ "github.com/pkg/errors"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+)
+
+// BinlogValidationResult contains detailed results of binlog access validation
+type BinlogValidationResult struct {
+ // Core validation state
+ Valid bool
+ Error error
+
+ // Detailed findings for specific error messages
+ BinlogEnabled bool
+ BinlogFormat string
+ HasPrivilege bool
+ MissingPrivileges []string // Specific privileges that are missing
+ CurrentGrants []string // Current grants for debugging
+}
+
+// ValidateBinlogAccess performs comprehensive validation of gh-ost prerequisites
+// Returns a structured result that can be used for both plan checks and execution
+func ValidateBinlogAccess(ctx context.Context, driver db.Driver, adminDataSource *storepb.DataSource) *BinlogValidationResult {
+ result := &BinlogValidationResult{
+ Valid: true,
+ BinlogEnabled: false,
+ HasPrivilege: false,
+ MissingPrivileges: []string{},
+ CurrentGrants: []string{},
+ }
+
+ // Test 1: Check if we can access binary log status
+ // Try both old and new MySQL commands for compatibility
+ canAccessBinlog := false
+ if _, err := driver.GetDB().ExecContext(ctx, "SHOW MASTER STATUS"); err == nil {
+ canAccessBinlog = true
+ } else if _, err := driver.GetDB().ExecContext(ctx, "SHOW BINARY LOG STATUS"); err == nil {
+ canAccessBinlog = true
+ }
+
+ if !canAccessBinlog {
+ result.Valid = false
+ result.Error = errors.New("cannot access binary logs - ensure user has REPLICATION CLIENT privilege")
+ slog.Error("binlog access validation failed: cannot access binary logs",
+ slog.String("host", adminDataSource.GetHost()),
+ slog.String("user", adminDataSource.GetUsername()))
+ return result
+ }
+
+ // Test 2: Check if binary logging is enabled
+ var logBin string
+ row := driver.GetDB().QueryRowContext(ctx, "SELECT @@log_bin")
+ if err := row.Scan(&logBin); err != nil {
+ result.Valid = false
+ result.Error = errors.Wrap(err, "failed to check if binary logging is enabled")
+ return result
+ }
+
+ result.BinlogEnabled = (logBin == "1" || strings.ToUpper(logBin) == "ON")
+ if !result.BinlogEnabled {
+ result.Valid = false
+ result.Error = errors.New("binary logging is not enabled on this MySQL instance")
+ return result
+ }
+
+ // Test 3: Check user privileges
+ rows, err := driver.GetDB().QueryContext(ctx, "SHOW GRANTS")
+ if err != nil {
+ result.Valid = false
+ result.Error = errors.Wrap(err, "failed to check user grants")
+ return result
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var grant string
+ if err := rows.Scan(&grant); err != nil {
+ slog.Warn("failed to scan grant",
+ slog.String("host", adminDataSource.GetHost()),
+ slog.String("user", adminDataSource.GetUsername()),
+ slog.String("error", err.Error()))
+ continue
+ }
+ result.CurrentGrants = append(result.CurrentGrants, grant)
+
+ upperGrant := strings.ToUpper(grant)
+ if strings.Contains(upperGrant, "REPLICATION SLAVE") ||
+ strings.Contains(upperGrant, "ALL PRIVILEGES") {
+ result.HasPrivilege = true
+ }
+ }
+
+ if err := rows.Err(); err != nil {
+ result.Valid = false
+ result.Error = errors.Wrap(err, "error reading grants")
+ return result
+ }
+
+ if !result.HasPrivilege {
+ result.Valid = false
+ result.MissingPrivileges = append(result.MissingPrivileges, "REPLICATION SLAVE")
+ result.Error = errors.New("user does not have REPLICATION SLAVE privilege required for gh-ost")
+ slog.Error("missing REPLICATION SLAVE privilege",
+ slog.String("host", adminDataSource.GetHost()),
+ slog.String("user", adminDataSource.GetUsername()),
+ slog.Any("grants", result.CurrentGrants))
+ return result
+ }
+
+ // Test 4: Check binlog format (must be ROW or MIXED, not STATEMENT)
+ row = driver.GetDB().QueryRowContext(ctx, "SELECT @@binlog_format")
+ if err := row.Scan(&result.BinlogFormat); err != nil {
+ result.Valid = false
+ result.Error = errors.Wrap(err, "failed to check binlog format")
+ return result
+ }
+
+ if strings.ToUpper(result.BinlogFormat) == "STATEMENT" {
+ result.Valid = false
+ result.Error = errors.Errorf("binlog_format is %s, but gh-ost requires ROW or MIXED format", result.BinlogFormat)
+ return result
+ }
+
+ // All checks passed
+ slog.Info("binlog access validation passed",
+ slog.String("host", adminDataSource.GetHost()),
+ slog.String("user", adminDataSource.GetUsername()),
+ slog.String("binlog_format", result.BinlogFormat))
+
+ return result
+}
+
+// GetUserFriendlyError returns a user-friendly error message based on validation results
+func (r *BinlogValidationResult) GetUserFriendlyError() (title, content string) {
+ if r.Valid {
+ return "", ""
+ }
+
+ title = "gh-ost migration prerequisites not met"
+
+ if !r.BinlogEnabled {
+ content = "Binary logging is not enabled on this MySQL instance. Please enable it with:\n" +
+ "SET GLOBAL log_bin=ON (requires MySQL restart)"
+ return title, content
+ }
+
+ if !r.HasPrivilege && len(r.MissingPrivileges) > 0 {
+ content = fmt.Sprintf("Database user is missing required privilege: %s\n", strings.Join(r.MissingPrivileges, ", ")) +
+ "Please grant it with:\n" +
+ "GRANT REPLICATION SLAVE ON *.* TO 'user'@'host'"
+ return title, content
+ }
+
+ if r.BinlogFormat == "STATEMENT" {
+ content = fmt.Sprintf("Current binlog_format is %s, but gh-ost requires ROW or MIXED format.\n", r.BinlogFormat) +
+ "Please change it with:\n" +
+ "SET GLOBAL binlog_format='ROW'"
+ return title, content
+ }
+
+ // Generic error fallback
+ if r.Error != nil {
+ content = fmt.Sprintf("Validation failed: %v", r.Error)
+ } else {
+ content = "Unknown validation error occurred"
+ }
+
+ return title, content
+}
diff --git a/backend/runner/plancheck/ghost_sync_executor.go b/backend/runner/plancheck/ghost_sync_executor.go
index 6d204bbdce..cbd8a86ebe 100644
--- a/backend/runner/plancheck/ghost_sync_executor.go
+++ b/backend/runner/plancheck/ghost_sync_executor.go
@@ -107,6 +107,34 @@ func (e *GhostSyncExecutor) Run(ctx context.Context, config *storepb.PlanCheckRu
return nil, common.Wrapf(err, common.Internal, "failed to parse table name from statement, statement: %v", statement)
}
+ // Validate binlog access before attempting migration
+ // This prevents retry storms and provides early feedback in plan checks
+ driver, err := e.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{})
+ if err != nil {
+ return []*storepb.PlanCheckRunResult_Result{
+ {
+ Status: storepb.PlanCheckRunResult_Result_ERROR,
+ Title: "Failed to connect to database",
+ Content: fmt.Sprintf("Cannot establish connection: %v", err),
+ Code: common.Internal.Int32(),
+ },
+ }, nil
+ }
+ defer driver.Close(ctx)
+
+ validationResult := ghost.ValidateBinlogAccess(ctx, driver, adminDataSource)
+ if !validationResult.Valid {
+ title, content := validationResult.GetUserFriendlyError()
+ return []*storepb.PlanCheckRunResult_Result{
+ {
+ Status: storepb.PlanCheckRunResult_Result_ERROR,
+ Title: title,
+ Content: content,
+ Code: common.Internal.Int32(),
+ },
+ }, nil
+ }
+
migrationContext, err := ghost.NewMigrationContext(ctx, rand.Intn(10000000), database, adminDataSource, tableName, fmt.Sprintf("_dryrun_%d", time.Now().Unix()), statement, true, config.GhostFlags, 20000000)
if err != nil {
return nil, common.Wrapf(err, common.Internal, "failed to create migration context")
@@ -122,12 +150,14 @@ func (e *GhostSyncExecutor) Run(ctx context.Context, config *storepb.PlanCheckRu
defer func() {
if err := func() error {
- ctx := context.Background()
- driver, err := e.dbFactory.GetAdminDatabaseDriver(ctx, instance, database, db.ConnectionContext{})
+ // Note: We're reusing the ctx from parent scope and creating a new driver
+ // because the original driver was already closed
+ cleanupCtx := context.Background()
+ cleanupDriver, err := e.dbFactory.GetAdminDatabaseDriver(cleanupCtx, instance, database, db.ConnectionContext{})
if err != nil {
- return errors.Wrapf(err, "failed to get driver")
+ return errors.Wrapf(err, "failed to get driver for cleanup")
}
- defer driver.Close(ctx)
+ defer cleanupDriver.Close(cleanupCtx)
// Use the backup database name of MySQL as the ghost database name.
ghostDBName := common.BackupDatabaseNameOfEngine(storepb.Engine_MYSQL)
@@ -138,7 +168,7 @@ func (e *GhostSyncExecutor) Run(ctx context.Context, config *storepb.PlanCheckRu
migrationContext.GetChangelogTableName(),
)
- if _, err := driver.GetDB().ExecContext(ctx, sql); err != nil {
+ if _, err := cleanupDriver.GetDB().ExecContext(cleanupCtx, sql); err != nil {
return errors.Wrapf(err, "failed to drop gh-ost temp tables")
}
return nil
diff --git a/go.mod b/go.mod
index 12bd204a89..cebe41f0b6 100644
--- a/go.mod
+++ b/go.mod
@@ -402,7 +402,7 @@ replace (
// fix potential security issue(CVE-2020-26160) introduced by indirect dependency.
github.com/dgrijalva/jwt-go => github.com/form3tech-oss/jwt-go v3.2.6-0.20210809144907-32ab6a8243d7+incompatible
// Other fixes.
- github.com/github/gh-ost => github.com/bytebase/gh-ost2 v1.1.7-0.20250310031106-33e16ca4b2e0
+ github.com/github/gh-ost => github.com/bytebase/gh-ost2 v1.1.7-0.20251002210738-35e5dddaad7c
github.com/jackc/pgx/v5 => github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767
diff --git a/go.sum b/go.sum
index 460d0c6eb1..1b01353346 100644
--- a/go.sum
+++ b/go.sum
@@ -809,8 +809,8 @@ github.com/bytebase/cosmosdb-parser v0.0.0-20250317064006-c1dcb3ed4fa4 h1:mw6ze5
github.com/bytebase/cosmosdb-parser v0.0.0-20250317064006-c1dcb3ed4fa4/go.mod h1:sivaOoKlMrXkuude2VGsKv6p83CRRGgblNfeVKQka/Y=
github.com/bytebase/doris-parser v0.0.0-20250512080703-bacdebf2f435 h1:Kr/zjp7arILvloH8IPnBYz8K7ntJxhRwLFnUa2m2z0E=
github.com/bytebase/doris-parser v0.0.0-20250512080703-bacdebf2f435/go.mod h1:mfKfCbkm5QFVKqZIwRxlUaY26kR3ttYeCiDLV4aUg7k=
-github.com/bytebase/gh-ost2 v1.1.7-0.20250310031106-33e16ca4b2e0 h1:wm4Set2lwArmC4f9hYdiunXtEYYzr6SyYRepTXNdkm4=
-github.com/bytebase/gh-ost2 v1.1.7-0.20250310031106-33e16ca4b2e0/go.mod h1:X0CGYwIZUnjfrv02/nBlftxqSBMm4mYJzFwiyn2D4d8=
+github.com/bytebase/gh-ost2 v1.1.7-0.20251002210738-35e5dddaad7c h1:scJ4ycKuUAaA5XMMq9/SQdWuupG6flnlq7v3Z65X0Dk=
+github.com/bytebase/gh-ost2 v1.1.7-0.20251002210738-35e5dddaad7c/go.mod h1:X0CGYwIZUnjfrv02/nBlftxqSBMm4mYJzFwiyn2D4d8=
github.com/bytebase/go-mssqldb v0.0.0-20240801091126-3ff3ca07d898 h1:0ggqA9XL4yvtePPvRDRYMed3jtdNk4s0o8AMW8uJPSA=
github.com/bytebase/go-mssqldb v0.0.0-20240801091126-3ff3ca07d898/go.mod h1:2KhUz4GSLcdJGXSJMXAf2BjmakcgoOkNyPpHdGAxRpY=
github.com/bytebase/gohive v0.0.0-20240422092929-d76993a958a4 h1:5o+GWOu+j68eAODMw7/lFJavWCCiPMeqEHuOGx0YmoU=