Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(parser/mysql): migrate backup, restore to omni AST; delete generic_checker [BYT-9119, BYT-9120] (#19848)

rebelice7fbce68ef086
diff --git a/backend/plugin/advisor/mysql/generic_checker.go b/backend/plugin/advisor/mysql/generic_checker.go
deleted file mode 100644
index b9ab3f666d..0000000000
--- a/backend/plugin/advisor/mysql/generic_checker.go
+++ /dev/null
@@ -1,173 +0,0 @@
-package mysql
-
-import (
- "fmt"
- "reflect"
- "strings"
-
- "github.com/antlr4-go/antlr/v4"
- "github.com/bytebase/parser/mysql"
-
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
-)
-
-// Node type constants for consistent node type checking
-const (
- NodeTypeCreateTable = "CreateTable"
- NodeTypeAlterTable = "AlterTable"
- NodeTypeAlterStatement = "AlterStatement"
- NodeTypeDropTable = "DropTable"
- NodeTypeRenameTableStatement = "RenameTableStatement"
- NodeTypeSetStatement = "SetStatement"
- NodeTypeCreateIndex = "CreateIndex"
- NodeTypeDropIndex = "DropIndex"
- NodeTypeInsertStatement = "InsertStatement"
- NodeTypeUpdateStatement = "UpdateStatement"
- NodeTypeDeleteStatement = "DeleteStatement"
- NodeTypeSelectStatement = "SelectStatement"
- NodeTypeCreateView = "CreateView"
- NodeTypeDropView = "DropView"
- NodeTypeCreateProcedure = "CreateProcedure"
- NodeTypeDropProcedure = "DropProcedure"
- NodeTypeCreateFunction = "CreateFunction"
- NodeTypeDropFunction = "DropFunction"
- NodeTypeCreateEvent = "CreateEvent"
- NodeTypeDropEvent = "DropEvent"
- NodeTypeCreateTrigger = "CreateTrigger"
- NodeTypeDropTrigger = "DropTrigger"
- NodeTypeQuery = "Query"
- NodeTypeQueryExpression = "QueryExpression"
- NodeTypeFunctionCall = "FunctionCall"
- NodeTypeCreateDatabase = "CreateDatabase"
- NodeTypeAlterDatabase = "AlterDatabase"
- NodeTypeDropDatabase = "DropDatabase"
- NodeTypeTruncateTableStatement = "TruncateTableStatement"
- NodeTypeSelectStatementWithInto = "SelectStatementWithInto"
- NodeTypeSelectItemList = "SelectItemList"
- NodeTypePureIdentifier = "PureIdentifier"
- NodeTypeIdentifierKeyword = "IdentifierKeyword"
- NodeTypeTransactionStatement = "TransactionStatement"
- NodeTypePredicateExprLike = "PredicateExprLike"
- NodeTypeSimpleStatement = "SimpleStatement"
- NodeTypeQuerySpecification = "QuerySpecification"
- NodeTypeLimitClause = "LimitClause"
- NodeTypeFromClause = "FromClause"
- NodeTypePrimaryExprCompare = "PrimaryExprCompare"
- NodeTypeJoinedTable = "JoinedTable"
- NodeTypeWhereClause = "WhereClause"
- NodeTypePredicateExprIn = "PredicateExprIn"
- NodeTypeExprList = "ExprList"
- NodeTypeExprOr = "ExprOr"
- NodeTypeAlterTableActions = "AlterTableActions"
-)
-
-// Rule defines the interface for individual SQL validation rules.
-// Each rule implements specific checking logic without embedding the base listener.
-type Rule interface {
- // OnEnter is called when entering a parse tree node
- OnEnter(ctx antlr.ParserRuleContext, nodeType string) error
-
- // OnExit is called when exiting a parse tree node
- OnExit(ctx antlr.ParserRuleContext, nodeType string) error
-
- // Name returns the rule name for logging/debugging
- Name() string
-
- // GetAdviceList returns the accumulated advice from this rule
- GetAdviceList() []*storepb.Advice
-}
-
-// GenericChecker embeds the base MySQL parser listener and dispatches events to registered rules.
-// This design ensures only one copy of the listener type metadata in the binary.
-type GenericChecker struct {
- *mysql.BaseMySQLParserListener
-
- rules []Rule
- baseLine int
-}
-
-// NewGenericChecker creates a new instance of GenericChecker with the given rules.
-func NewGenericChecker(rules []Rule) *GenericChecker {
- return &GenericChecker{
- rules: rules,
- }
-}
-
-// SetBaseLine sets the base line number for error reporting.
-func (g *GenericChecker) SetBaseLine(baseLine int) {
- g.baseLine = baseLine
-}
-
-// GetBaseLine returns the current base line number.
-func (g *GenericChecker) GetBaseLine() int {
- return g.baseLine
-}
-
-// EnterEveryRule is called when any rule is entered.
-// It dispatches the event to all registered rules.
-func (g *GenericChecker) EnterEveryRule(ctx antlr.ParserRuleContext) {
- nodeType := g.getNodeType(ctx)
- for _, rule := range g.rules {
- if err := rule.OnEnter(ctx, nodeType); err != nil {
- // Log error if needed
- fmt.Printf("Rule %s error on enter %s: %v\n", rule.Name(), nodeType, err)
- }
- }
-}
-
-// ExitEveryRule is called when any rule is exited.
-// It dispatches the event to all registered rules.
-func (g *GenericChecker) ExitEveryRule(ctx antlr.ParserRuleContext) {
- nodeType := g.getNodeType(ctx)
- for _, rule := range g.rules {
- if err := rule.OnExit(ctx, nodeType); err != nil {
- // Log error if needed
- fmt.Printf("Rule %s error on exit %s: %v\n", rule.Name(), nodeType, err)
- }
- }
-}
-
-// GetAdviceList collects and returns all advice from registered rules.
-func (g *GenericChecker) GetAdviceList() []*storepb.Advice {
- var allAdvice []*storepb.Advice
- for _, rule := range g.rules {
- allAdvice = append(allAdvice, rule.GetAdviceList()...)
- }
- return allAdvice
-}
-
-// getNodeType returns the type name of the parse tree node.
-func (*GenericChecker) getNodeType(ctx antlr.ParserRuleContext) string {
- t := reflect.TypeOf(ctx)
- if t.Kind() == reflect.Ptr {
- t = t.Elem()
- }
- name := t.Name()
- // Remove "Context" suffix if present
- name = strings.TrimSuffix(name, "Context")
- return name
-}
-
-// BaseRule provides common functionality for rules.
-// Other rules can embed this struct to get common behavior.
-type BaseRule struct {
- level storepb.Advice_Status // nolint:unused
- title string // nolint:unused
- adviceList []*storepb.Advice
- baseLine int
-}
-
-// SetBaseLine sets the base line for the rule.
-func (r *BaseRule) SetBaseLine(baseLine int) {
- r.baseLine = baseLine
-}
-
-// GetAdviceList returns the accumulated advice.
-func (r *BaseRule) GetAdviceList() []*storepb.Advice {
- return r.adviceList
-}
-
-// AddAdvice adds a new advice to the list.
-func (r *BaseRule) AddAdvice(advice *storepb.Advice) {
- r.adviceList = append(r.adviceList, advice)
-}
diff --git a/backend/plugin/advisor/mysql/rule_builtin_prior_backup_check.go b/backend/plugin/advisor/mysql/rule_builtin_prior_backup_check.go
index 23d27037ea..3a0a8e3d3f 100644
--- a/backend/plugin/advisor/mysql/rule_builtin_prior_backup_check.go
+++ b/backend/plugin/advisor/mysql/rule_builtin_prior_backup_check.go
@@ -117,11 +117,11 @@ func prepareTransformation(databaseName string, parsedStatements []base.ParsedSt
if stmt.AST == nil {
continue
}
- ast, ok := base.GetANTLRAST(stmt.AST)
+ node, ok := mysqlparser.GetOmniNode(stmt.AST)
if !ok {
continue
}
- tables, err := mysqlparser.ExtractTables(databaseName, ast, i, nil)
+ tables, err := mysqlparser.ExtractTables(databaseName, node, stmt.Text, i, nil)
if err != nil {
return nil, errors.Wrap(err, "failed to extract tables")
}
diff --git a/backend/plugin/parser/mysql/backup.go b/backend/plugin/parser/mysql/backup.go
index b27c6ef3ac..5fa119d984 100644
--- a/backend/plugin/parser/mysql/backup.go
+++ b/backend/plugin/parser/mysql/backup.go
@@ -7,42 +7,24 @@ import (
"slices"
"strings"
- "github.com/antlr4-go/antlr/v4"
"github.com/pkg/errors"
- parser "github.com/bytebase/parser/mysql"
+ "github.com/bytebase/omni/mysql/ast"
"github.com/bytebase/bytebase/backend/common"
- "github.com/bytebase/bytebase/backend/generated-go/store"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store/model"
)
func init() {
- base.RegisterTransformDMLToSelect(store.Engine_MYSQL, TransformDMLToSelect)
+ base.RegisterTransformDMLToSelect(storepb.Engine_MYSQL, TransformDMLToSelect)
}
const (
maxTableNameLength = 64
)
-func TransformDMLToSelect(ctx context.Context, tCtx base.TransformContext, statement string, sourceDatabase string, targetDatabase string, tablePrefix string) ([]base.BackupStatement, error) {
- var dbMetadata *model.DatabaseMetadata
- if tCtx.GetDatabaseMetadataFunc != nil {
- _, metadata, err := tCtx.GetDatabaseMetadataFunc(ctx, tCtx.InstanceID, sourceDatabase)
- if err == nil {
- dbMetadata = metadata
- }
- }
-
- statementInfoList, err := prepareTransformation(sourceDatabase, statement, dbMetadata)
- if err != nil {
- return nil, errors.Wrap(err, "failed to prepare transformation")
- }
-
- return generateSQL(ctx, tCtx, statementInfoList, targetDatabase, tablePrefix)
-}
-
type StatementType int
const (
@@ -62,10 +44,28 @@ type TableReference struct {
type StatementInfo struct {
Offset int
Statement string
- Tree antlr.ParserRuleContext
+ Node ast.Node
Table *TableReference
- StartPosition *store.Position
- EndPosition *store.Position
+ StartPosition *storepb.Position
+ EndPosition *storepb.Position
+ FullSQL string
+}
+
+func TransformDMLToSelect(ctx context.Context, tCtx base.TransformContext, statement string, sourceDatabase string, targetDatabase string, tablePrefix string) ([]base.BackupStatement, error) {
+ var dbMetadata *model.DatabaseMetadata
+ if tCtx.GetDatabaseMetadataFunc != nil {
+ _, metadata, err := tCtx.GetDatabaseMetadataFunc(ctx, tCtx.InstanceID, sourceDatabase)
+ if err == nil {
+ dbMetadata = metadata
+ }
+ }
+
+ statementInfoList, err := prepareTransformation(sourceDatabase, statement, dbMetadata)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to prepare transformation")
+ }
+
+ return generateSQL(ctx, tCtx, statementInfoList, targetDatabase, tablePrefix)
}
func prepareTransformation(databaseName, statement string, dbMetadata *model.DatabaseMetadata) ([]StatementInfo, error) {
@@ -75,32 +75,33 @@ func prepareTransformation(databaseName, statement string, dbMetadata *model.Dat
}
var result []StatementInfo
-
for i, item := range list {
if len(item.Text) == 0 || item.Empty {
continue
}
- parseResult, err := ParseMySQL(item.Text)
+
+ parsed, err := ParseMySQLOmni(item.Text)
if err != nil {
return nil, errors.Wrap(err, "failed to parse sql")
}
+ if parsed == nil || len(parsed.Items) == 0 {
+ continue
+ }
- for _, sql := range parseResult {
- // After splitting the SQL, we should have only one statement in the list.
- // The FOR loop is just for safety.
- // So we can use the i as the offset.
- tables, err := ExtractTables(databaseName, sql, i, dbMetadata)
+ for _, node := range parsed.Items {
+ tables, err := ExtractTables(databaseName, node, item.Text, i, dbMetadata)
if err != nil {
return nil, errors.Wrap(err, "failed to extract tables")
}
for _, table := range tables {
result = append(result, StatementInfo{
- Offset: i,
+ Offset: table.Offset,
Statement: table.Statement,
+ Node: table.Node,
Table: table.Table,
- Tree: table.Tree,
StartPosition: item.Start,
EndPosition: item.End,
+ FullSQL: item.Text,
})
}
}
@@ -109,6 +110,367 @@ func prepareTransformation(databaseName, statement string, dbMetadata *model.Dat
return result, nil
}
+// ExtractTables extracts table references from a DML statement.
+func ExtractTables(databaseName string, node ast.Node, fullSQL string, offset int, dbMetadata *model.DatabaseMetadata) ([]StatementInfo, error) {
+ switch n := node.(type) {
+ case *ast.DeleteStmt:
+ return extractTablesFromDelete(databaseName, n, fullSQL, offset, dbMetadata)
+ case *ast.UpdateStmt:
+ return extractTablesFromUpdate(databaseName, n, fullSQL, offset, dbMetadata)
+ default:
+ return nil, nil
+ }
+}
+
+func extractTablesFromDelete(databaseName string, n *ast.DeleteStmt, fullSQL string, offset int, _ *model.DatabaseMetadata) ([]StatementInfo, error) {
+ cteNames := collectCTENames(fullSQL, n.Loc)
+ stmtText := extractStatementText(fullSQL, n.Loc)
+
+ // Collect all single table refs from the Tables list.
+ singleTables := collectSingleTables(databaseName, n.Tables)
+
+ if len(n.Using) == 0 {
+ // Single-table DELETE: DELETE FROM t WHERE ...
+ if len(singleTables) == 0 {
+ return nil, nil
+ }
+ for _, ref := range singleTables {
+ ref.StatementType = StatementTypeDelete
+ }
+ first := singleTables[firstKey(singleTables)]
+ return []StatementInfo{{
+ Offset: offset,
+ Statement: stmtText,
+ Node: n,
+ Table: first,
+ FullSQL: fullSQL,
+ }}, nil
+ }
+
+ // Multi-table DELETE: DELETE t1, t2 FROM t1 JOIN t2 ... WHERE ...
+ // Tables = targets to delete from, Using = referenced table list.
+ refTables := collectSingleTables(databaseName, n.Using)
+
+ var result []StatementInfo
+ for _, target := range singleTables {
+ name := target.Table
+ if target.Alias != "" {
+ name = target.Alias
+ }
+ if cteNames[name] {
+ continue
+ }
+ ref, ok := refTables[name]
+ if !ok {
+ return nil, errors.Errorf("cannot extract reference table: no matched table %q in referenced table list", name)
+ }
+ ref.StatementType = StatementTypeDelete
+ result = append(result, StatementInfo{
+ Offset: offset,
+ Statement: stmtText,
+ Node: n,
+ Table: ref,
+ FullSQL: fullSQL,
+ })
+ }
+ return result, nil
+}
+
+func extractTablesFromUpdate(databaseName string, n *ast.UpdateStmt, fullSQL string, offset int, dbMetadata *model.DatabaseMetadata) ([]StatementInfo, error) {
+ cteNames := collectCTENames(fullSQL, n.Loc)
+ stmtText := extractStatementText(fullSQL, n.Loc)
+ singleTables := collectSingleTables(databaseName, n.Tables)
+
+ // Determine which tables are updated via SET clause column prefixes.
+ updatedTables := make(map[string]bool)
+ var unqualifiedColumns []string
+ for _, assign := range n.SetList {
+ if assign.Column != nil {
+ table := assign.Column.Table
+ if !cteNames[table] {
+ updatedTables[table] = true
+ if table == "" {
+ unqualifiedColumns = append(unqualifiedColumns, assign.Column.Column)
+ }
+ }
+ }
+ }
+
+ // Resolve unqualified columns using metadata.
+ if updatedTables[""] {
+ delete(updatedTables, "")
+ resolved := resolveUnqualifiedColumns(unqualifiedColumns, singleTables, dbMetadata)
+ for t := range resolved {
+ updatedTables[t] = true
+ }
+ }
+
+ // Single-table UPDATE (only one table, no explicit qualification needed).
+ if len(updatedTables) == 0 && len(singleTables) == 1 {
+ for _, ref := range singleTables {
+ ref.StatementType = StatementTypeUpdate
+ return []StatementInfo{{
+ Offset: offset,
+ Statement: stmtText,
+ Node: n,
+ Table: ref,
+ FullSQL: fullSQL,
+ }}, nil
+ }
+ }
+
+ var result []StatementInfo
+ for table := range updatedTables {
+ ref, ok := singleTables[table]
+ if !ok {
+ return nil, errors.Errorf("cannot extract reference table: no matched updated table %q in referenced table list", table)
+ }
+ ref.StatementType = StatementTypeUpdate
+ result = append(result, StatementInfo{
+ Offset: offset,
+ Statement: stmtText,
+ Node: n,
+ Table: ref,
+ FullSQL: fullSQL,
+ })
+ }
+ return result, nil
+}
+
+// collectSingleTables walks TableExpr slices and collects TableRef entries keyed by alias or name.
+func collectSingleTables(databaseName string, exprs []ast.TableExpr) map[string]*TableReference {
+ result := make(map[string]*TableReference)
+ for _, expr := range exprs {
+ collectSingleTablesFromExpr(databaseName, expr, result)
+ }
+ return result
+}
+
+func collectSingleTablesFromExpr(databaseName string, expr ast.TableExpr, out map[string]*TableReference) {
+ switch e := expr.(type) {
+ case *ast.TableRef:
+ db := e.Schema
+ if db == "" {
+ db = databaseName
+ }
+ ref := &TableReference{
+ Database: db,
+ Table: e.Name,
+ Alias: e.Alias,
+ }
+ key := e.Name
+ if e.Alias != "" {
+ key = e.Alias
+ }
+ out[key] = ref
+ case *ast.JoinClause:
+ if e.Left != nil {
+ collectSingleTablesFromExpr(databaseName, e.Left, out)
+ }
+ if e.Right != nil {
+ collectSingleTablesFromExpr(databaseName, e.Right, out)
+ }
+ default:
+ }
+}
+
+// collectCTENames extracts CTE names from the SQL text before the statement Loc.
+// MySQL's UPDATE/DELETE CTEs are not in the omni AST, so we parse the prefix.
+func collectCTENames(fullSQL string, stmtLoc ast.Loc) map[string]bool {
+ result := make(map[string]bool)
+ for _, cte := range parseCTEPrefix(fullSQL, stmtLoc) {
+ result[cte.Name] = true
+ }
+ return result
+}
+
+// extractCTE extracts the CTE (WITH clause) text from the SQL before the statement.
+func extractCTE(fullSQL string, stmtLoc ast.Loc) string {
+ if stmtLoc.Start <= 0 {
+ return ""
+ }
+ ctes := parseCTEPrefix(fullSQL, stmtLoc)
+ if len(ctes) == 0 {
+ return ""
+ }
+ return strings.TrimSpace(fullSQL[:stmtLoc.Start])
+}
+
+// parseCTEPrefix parses the SQL text before stmtLoc as a WITH clause.
+// Returns the CTE list if the prefix contains a valid WITH clause, nil otherwise.
+func parseCTEPrefix(fullSQL string, stmtLoc ast.Loc) []*ast.CommonTableExpr {
+ if stmtLoc.Start <= 0 {
+ return nil
+ }
+ prefix := strings.TrimSpace(fullSQL[:stmtLoc.Start])
+ if prefix == "" {
+ return nil
+ }
+ // Wrap the prefix with a dummy SELECT to make it parseable.
+ parsed, err := ParseMySQLOmni(prefix + " SELECT 1")
+ if err != nil || parsed == nil {
+ return nil
+ }
+ for _, item := range parsed.Items {
+ if sel, ok := item.(*ast.SelectStmt); ok && len(sel.CTEs) > 0 {
+ return sel.CTEs
+ }
+ }
+ return nil
+}
+
+func extractStatementText(fullSQL string, loc ast.Loc) string {
+ end := loc.End
+ if end <= 0 || end > len(fullSQL) {
+ end = len(fullSQL)
+ }
+ start := 0
+ if loc.Start > 0 {
+ start = loc.Start
+ }
+ return strings.TrimSpace(fullSQL[start:end])
+}
+
+// writeSuffixSelectClause writes the FROM ... WHERE ... suffix of a DML statement.
+func writeSuffixSelectClause(buf *strings.Builder, node ast.Node, fullSQL string) error {
+ switch n := node.(type) {
+ case *ast.UpdateStmt:
+ return writeUpdateSuffix(buf, n, fullSQL)
+ case *ast.DeleteStmt:
+ return writeDeleteSuffix(buf, n, fullSQL)
+ }
+ return nil
+}
+
+func writeUpdateSuffix(buf *strings.Builder, n *ast.UpdateStmt, sql string) error {
+ // For UPDATE: write "table_refs WHERE ... ORDER BY ... LIMIT ..."
+ // Use the last table's Loc.End as the boundary — everything from first table
+ // to last table's end is the table reference list.
+ if len(n.Tables) > 0 {
+ start := nodeLocStart(n.Tables[0])
+ end := nodeLocEnd(n.Tables[len(n.Tables)-1])
+ if start >= 0 && end > start && end <= len(sql) {
+ if _, err := buf.WriteString(strings.TrimSpace(sql[start:end])); err != nil {
+ return err
+ }
+ }
+ }
+
+ // Everything after the last SET assignment is WHERE + ORDER BY + LIMIT.
+ if len(n.SetList) > 0 {
+ lastAssign := n.SetList[len(n.SetList)-1]
+ afterSet := lastAssign.Loc.End
+ stmtEnd := n.Loc.End
+ if stmtEnd <= 0 || stmtEnd > len(sql) {
+ stmtEnd = len(sql)
+ }
+ suffix := strings.TrimSpace(sql[afterSet:stmtEnd])
+ if suffix != "" {
+ if _, err := fmt.Fprintf(buf, " %s", suffix); err != nil {
+ return err
+ }
+ }
+ }
+
+ return nil
+}
+
+func writeDeleteSuffix(buf *strings.Builder, n *ast.DeleteStmt, sql string) error {
+ // For single-table DELETE: everything from the table ref to statement end.
+ // For multi-table DELETE: everything from the USING/reference table list to end.
+ if len(n.Using) > 0 {
+ start := nodeLocStart(n.Using[0])
+ end := n.Loc.End
+ if end <= 0 || end > len(sql) {
+ end = len(sql)
+ }
+ if start >= 0 && end > start {
+ if _, err := buf.WriteString(strings.TrimSpace(sql[start:end])); err != nil {
+ return err
+ }
+ }
+ } else if len(n.Tables) > 0 {
+ start := nodeLocStart(n.Tables[0])
+ end := n.Loc.End
+ if end <= 0 || end > len(sql) {
+ end = len(sql)
+ }
+ if start >= 0 && end > start {
+ if _, err := buf.WriteString(strings.TrimSpace(sql[start:end])); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+func nodeLocStart(expr ast.TableExpr) int {
+ switch e := expr.(type) {
+ case *ast.TableRef:
+ return e.Loc.Start
+ case *ast.JoinClause:
+ return e.Loc.Start
+ }
+ return -1
+}
+
+func nodeLocEnd(expr ast.TableExpr) int {
+ switch e := expr.(type) {
+ case *ast.TableRef:
+ return e.Loc.End
+ case *ast.JoinClause:
+ return e.Loc.End
+ }
+ return -1
+}
+
+// resolveUnqualifiedColumns resolves unqualified column names to owning tables using metadata.
+func resolveUnqualifiedColumns(columns []string, singleTables map[string]*TableReference, dbMetadata *model.DatabaseMetadata) map[string]bool {
+ result := make(map[string]bool)
+
+ if dbMetadata == nil {
+ for t := range singleTables {
+ result[t] = true
+ }
+ return result
+ }
+ schema := dbMetadata.GetSchemaMetadata("")
+ if schema == nil {
+ for t := range singleTables {
+ result[t] = true
+ }
+ return result
+ }
+
+ for _, col := range columns {
+ resolved := false
+ for aliasOrName, ref := range singleTables {
+ tableMetadata := schema.GetTable(ref.Table)
+ if tableMetadata == nil {
+ continue
+ }
+ if tableMetadata.GetColumn(col) != nil {
+ result[aliasOrName] = true
+ resolved = true
+ }
+ }
+ if !resolved {
+ for t := range singleTables {
+ result[t] = true
+ }
+ return result
+ }
+ }
+ return result
+}
+
+func firstKey(m map[string]*TableReference) string {
+ for k := range m {
+ return k
+ }
+ return ""
+}
+
func generateSQL(ctx context.Context, tCtx base.TransformContext, statementInfoList []StatementInfo, databaseName string, tablePrefix string) ([]base.BackupStatement, error) {
groupByTable := make(map[string][]StatementInfo)
for _, item := range statementInfoList {
@@ -116,7 +478,6 @@ func generateSQL(ctx context.Context, tCtx base.TransformContext, statementInfoL
groupByTable[key] = append(groupByTable[key], item)
}
- // Check if the statement type is the same for all statements in one table.
for key, list := range groupByTable {
stmtType := StatementTypeUnknown
for _, item := range list {
@@ -130,7 +491,6 @@ func generateSQL(ctx context.Context, tCtx base.TransformContext, statementInfoL
}
var result []base.BackupStatement
-
for key, list := range groupByTable {
backupStatement, err := generateSQLForTable(ctx, tCtx, list, databaseName, tablePrefix)
if err != nil {
@@ -202,8 +562,6 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statem
}
for i, item := range statementInfoList {
if i != 0 {
- // We assume that the source table has a primary key.
- // If we have multiple statements, we need to use UNION DISTINCT to avoid duplicate rows.
if _, err := buf.WriteString("\n UNION DISTINCT\n"); err != nil {
return nil, errors.Wrap(err, "failed to write union all statement")
}
@@ -215,12 +573,15 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statem
if _, err := buf.WriteString(" "); err != nil {
return nil, errors.Wrap(err, "failed to write space")
}
- cteString := extractCTE(item.Tree)
+
+ stmtLoc := nodeStmtLoc(item.Node)
+ cteString := extractCTE(item.FullSQL, stmtLoc)
if len(cteString) > 0 {
if _, err := fmt.Fprintf(&buf, "%s ", cteString); err != nil {
return nil, errors.Wrap(err, "failed to write cte")
}
}
+
if len(generatedColumns) == 0 {
if _, err := fmt.Fprintf(&buf, "SELECT `%s`.* FROM ", tableNameOrAlias); err != nil {
return nil, errors.Wrap(err, "failed to write select statement")
@@ -229,8 +590,8 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statem
if _, err := buf.WriteString("SELECT "); err != nil {
return nil, errors.Wrap(err, "failed to write select statement")
}
- for i, column := range normalColumns {
- if i > 0 {
+ for j, column := range normalColumns {
+ if j > 0 {
if err := buf.WriteByte(','); err != nil {
return nil, errors.Wrap(err, "failed to write comma")
}
@@ -243,7 +604,7 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statem
return nil, errors.Wrap(err, "failed to write from")
}
}
- if err := extractSuffixSelectStatement(item.Tree, &buf); err != nil {
+ if err := writeSuffixSelectClause(&buf, item.Node, item.FullSQL); err != nil {
return nil, errors.Wrap(err, "failed to extract suffix select statement")
}
}
@@ -261,20 +622,14 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statem
}, nil
}
-func extractCTE(ctx antlr.ParserRuleContext) string {
- switch node := ctx.(type) {
- case *parser.UpdateStatementContext:
- if node.WithClause() != nil {
- return node.GetParser().GetTokenStream().GetTextFromRuleContext(node.WithClause())
- }
- case *parser.DeleteStatementContext:
- if node.WithClause() != nil {
- return node.GetParser().GetTokenStream().GetTextFromRuleContext(node.WithClause())
- }
- default:
+func nodeStmtLoc(node ast.Node) ast.Loc {
+ switch n := node.(type) {
+ case *ast.UpdateStmt:
+ return n.Loc
+ case *ast.DeleteStmt:
+ return n.Loc
}
-
- return ""
+ return ast.Loc{}
}
func classifyColumns(ctx context.Context, getDatabaseMetadataFunc base.GetDatabaseMetadataFunc, listDatabaseNamesFunc base.ListDatabaseNamesFunc, isCaseSensitive bool, instanceID string, table *TableReference) ([]string, []string, error) {
@@ -313,8 +668,7 @@ func classifyColumns(ctx context.Context, getDatabaseMetadataFunc base.GetDataba
return nil, nil, errors.Errorf("failed to get database metadata for InstanceID %q, Database %q", instanceID, table.Database)
}
- emptySchema := ""
- schema := dbMetadata.GetSchemaMetadata(emptySchema)
+ schema := dbMetadata.GetSchemaMetadata("")
if schema == nil {
return nil, nil, errors.New("failed to get schema metadata")
}
@@ -345,359 +699,3 @@ func classifyColumns(ctx context.Context, getDatabaseMetadataFunc base.GetDataba
return generatedColumns, normalColumns, nil
}
-
-func extractSuffixSelectStatement(tree antlr.Tree, buf *strings.Builder) error {
- listener := &suffixSelectStatementListener{
- buf: buf,
- }
-
- antlr.ParseTreeWalkerDefault.Walk(listener, tree)
- return listener.err
-}
-
-type suffixSelectStatementListener struct {
- *parser.BaseMySQLParserListener
-
- buf *strings.Builder
- err error
-}
-
-func (l *suffixSelectStatementListener) EnterDeleteStatement(ctx *parser.DeleteStatementContext) {
- if !isTopLevel(ctx.GetParent()) {
- return
- }
- if ctx.TableRef() != nil {
- // Single table delete statement.
- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromTokens(
- ctx.TableRef().GetStart(),
- ctx.GetStop(),
- )); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
- }
-
- if ctx.TableAliasRefList() != nil {
- // Multi table delete statement.
- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromTokens(
- ctx.TableReferenceList().GetStart(),
- ctx.GetStop(),
- )); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
- }
-}
-
-func (l *suffixSelectStatementListener) EnterUpdateStatement(ctx *parser.UpdateStatementContext) {
- if !isTopLevel(ctx.GetParent()) {
- return
- }
- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.TableReferenceList())); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
-
- if ctx.WhereClause() != nil {
- if err := l.buf.WriteByte(' '); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.WhereClause())); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
- }
-
- if ctx.OrderClause() != nil {
- if err := l.buf.WriteByte(' '); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.OrderClause())); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
- }
-
- if ctx.SimpleLimitClause() != nil {
- if err := l.buf.WriteByte(' '); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.SimpleLimitClause())); err != nil {
- l.err = errors.Wrap(err, "failed to write suffix select statement")
- return
- }
- }
-}
-
-func ExtractTables(databaseName string, ast *base.ANTLRAST, offset int, dbMetadata *model.DatabaseMetadata) ([]StatementInfo, error) {
- listener := &tableReferenceListener{
- databaseName: databaseName,
- dbMetadata: dbMetadata,
- offset: offset,
- }
-
- antlr.ParseTreeWalkerDefault.Walk(listener, ast.Tree)
-
- return listener.tables, listener.err
-}
-
-type tableReferenceListener struct {
- *parser.BaseMySQLParserListener
-
- databaseName string
- dbMetadata *model.DatabaseMetadata
- offset int
- tables []StatementInfo
- err error
-}
-
-func isTopLevel(ctx antlr.Tree) bool {
- if ctx == nil {
- return true
- }
- switch ctx := ctx.(type) {
- case *parser.SimpleStatementContext:
- return isTopLevel(ctx.GetParent())
- case *parser.QueryContext, *parser.ScriptContext:
- return true
- default:
- return false
- }
-}
-
-func (l *tableReferenceListener) EnterDeleteStatement(ctx *parser.DeleteStatementContext) {
- if !isTopLevel(ctx.GetParent()) {
- return
- }
-
- cteMap := make(map[string]bool)
- if ctx.WithClause() != nil {
- for _, cte := range ctx.WithClause().AllCommonTableExpression() {
- tableName := NormalizeMySQLIdentifier(cte.Identifier())
- cteMap[tableName] = true
- }
- }
-
- if ctx.TableRef() != nil {
- // Single table delete statement.
- database, table := NormalizeMySQLTableRef(ctx.TableRef())
- if len(database) > 0 && database != l.databaseName {
- l.err = errors.Errorf("database is not matched: %s != %s", database, l.databaseName)
- return
- }
-
- alias := ""
-
- if ctx.TableAlias() != nil {
- alias = NormalizeMySQLIdentifier(ctx.TableAlias().Identifier())
- }
-
- if len(database) == 0 {
- database = l.databaseName
- }
-
- l.tables = append(l.tables, StatementInfo{
- Offset: l.offset,
- Statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),
- Tree: ctx,
- Table: &TableReference{
- Database: database,
- Table: table,
- Alias: alias,
- StatementType: StatementTypeDelete,
- },
- })
- return
- }
-
- if ctx.TableAliasRefList() != nil {
- // Multi table delete statement.
- singleTables := &singleTableListener{
- databaseName: l.databaseName,
- singleTables: make(map[string]*TableReference),
- }
-
- antlr.ParseTreeWalkerDefault.Walk(singleTables, ctx.TableReferenceList())
-
- for _, tableRef := range ctx.TableAliasRefList().AllTableRefWithWildcard() {
- database, table := NormalizeMySQLTableRefWithWildcard(tableRef)
- if len(database) > 0 && database != l.databaseName {
- l.err = errors.Errorf("database is not matched: %s != %s", database, l.databaseName)
- return
- }
-
- if len(database) == 0 && cteMap[table] {
- continue
- }
-
- singleTable, ok := singleTables.singleTables[table]
- if !ok {
- l.err = errors.Errorf("cannot extract reference table: no matched table %q in referenced table list", table)
- return
- }
-
- singleTable.StatementType = StatementTypeDelete
- l.tables = append(l.tables, StatementInfo{
- Offset: l.offset,
- Statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),
- Tree: ctx,
- Table: singleTable,
- })
- }
- }
-}
-
-// resolveUnqualifiedColumns resolves unqualified column names to their owning
-// table(s) using database metadata. Falls back to all tables if metadata is
-// unavailable or column cannot be resolved.
-func (l *tableReferenceListener) resolveUnqualifiedColumns(columns []string, singleTables map[string]*TableReference) map[string]bool {
- result := make(map[string]bool)
-
- schema := l.getSchemaMetadata()
- if schema == nil {
- // No metadata available, fall back to all tables.
- for tableName := range singleTables {
- result[tableName] = true
- }
- return result
- }
-
- for _, col := range columns {
- resolved := false
- for aliasOrName, ref := range singleTables {
- tableMetadata := schema.GetTable(ref.Table)
- if tableMetadata == nil {
- continue
- }
- if tableMetadata.GetColumn(col) != nil {
- result[aliasOrName] = true
- resolved = true
- }
- }
- if !resolved {
- // Column not found in any table metadata, fall back to all tables.
- for tableName := range singleTables {
- result[tableName] = true
- }
- return result
- }
- }
-
- return result
-}
-
-func (l *tableReferenceListener) getSchemaMetadata() *model.SchemaMetadata {
- if l.dbMetadata == nil {
- return nil
- }
- return l.dbMetadata.GetSchemaMetadata("")
-}
-
-func (l *tableReferenceListener) EnterUpdateStatement(ctx *parser.UpdateStatementContext) {
- if !isTopLevel(ctx.GetParent()) {
- return
- }
-
- cteMap := make(map[string]bool)
-
- if ctx.WithClause() != nil {
- for _, cte := range ctx.WithClause().AllCommonTableExpression() {
- tableName := NormalizeMySQLIdentifier(cte.Identifier())
- cteMap[tableName] = true
- }
- }
-
- listener := &updateTableListener{
- tables: make(map[string]bool),
- cteMap: cteMap,
- }
-
- antlr.ParseTreeWalkerDefault.Walk(listener, ctx.UpdateList())
-
- singleTables := &singleTableListener{
- databaseName: l.databaseName,
- singleTables: make(map[string]*TableReference),
- }
-
- antlr.ParseTreeWalkerDefault.Walk(singleTables, ctx.TableReferenceList())
-
- // When SET clause columns are unqualified (no table prefix), resolve them
- // to the owning table using database metadata. MySQL allows unqualified
- // columns when they are unambiguous.
- if _, exists := listener.tables[""]; exists {
- delete(listener.tables, "")
- resolved := l.resolveUnqualifiedColumns(listener.unqualifiedColumns, singleTables.singleTables)
- for tableName := range resolved {
- listener.tables[tableName] = true
- }
- }
-
- for table := range listener.tables {
- singleTable, ok := singleTables.singleTables[table]
- if !ok {
- l.err = errors.Errorf("cannot extract reference table: no matched updated table %q in referenced table list", table)
- return
- }
-
- singleTable.StatementType = StatementTypeUpdate
- l.tables = append(l.tables, StatementInfo{
- Offset: l.offset,
- Statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),
- Tree: ctx,
- Table: singleTable,
- })
- }
-}
-
-type singleTableListener struct {
- *parser.BaseMySQLParserListener
-
- databaseName string
- singleTables map[string]*TableReference
- err error
-}
-
-func (l *singleTableListener) EnterSingleTable(ctx *parser.SingleTableContext) {
- if l.err != nil {
- return
- }
- database, tableName := NormalizeMySQLTableRef(ctx.TableRef())
- if len(database) > 0 && database != l.databaseName {
- l.err = errors.Errorf("database is not matched: %s != %s", database, l.databaseName)
- }
- if len(database) == 0 {
- database = l.databaseName
- }
- table := &TableReference{
- Database: database,
- Table: tableName,
- }
-
- if ctx.TableAlias() != nil {
- table.Alias = NormalizeMySQLIdentifier(ctx.TableAlias().Identifier())
- l.singleTables[table.Alias] = table
- } else {
- l.singleTables[table.Table] = table
- }
-}
-
-type updateTableListener struct {
- *parser.BaseMySQLParserListener
-
- tables map[string]bool
- unqualifiedColumns []string
- cteMap map[string]bool
-}
-
-func (l *updateTableListener) EnterUpdateElement(ctx *parser.UpdateElementContext) {
- _, table, column := NormalizeMySQLColumnRef(ctx.ColumnRef())
- if _, exists := l.cteMap[table]; !exists {
- l.tables[table] = true
- if table == "" {
- l.unqualifiedColumns = append(l.unqualifiedColumns, column)
- }
- }
-}
diff --git a/backend/plugin/parser/mysql/query_type.go b/backend/plugin/parser/mysql/query_type.go
index be421d754d..5b0b86e55c 100644
--- a/backend/plugin/parser/mysql/query_type.go
+++ b/backend/plugin/parser/mysql/query_type.go
@@ -1,17 +1,31 @@
package mysql
import (
+ "github.com/antlr4-go/antlr/v4"
"github.com/bytebase/parser/mysql"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
+func isTopLevel(ctx antlr.Tree) bool {
+ if ctx == nil {
+ return true
+ }
+ switch ctx := ctx.(type) {
+ case *mysql.SimpleStatementContext:
+ return isTopLevel(ctx.GetParent())
+ case *mysql.QueryContext, *mysql.ScriptContext:
+ return true
+ default:
+ return false
+ }
+}
+
type queryTypeListener struct {
*mysql.BaseMySQLParserListener
- allSystems bool
- result base.QueryType
- // This is for skipping the query span.
+ allSystems bool
+ result base.QueryType
isExplainAnalyze bool
}
@@ -42,7 +56,6 @@ func (l *queryTypeListener) EnterSimpleStatement(ctx *mysql.SimpleStatementConte
l.result = base.DML
case ctx.SetStatement() != nil:
if len(ctx.SetStatement().StartOptionValueList().AllPASSWORD_SYMBOL()) == 0 {
- // Treat SAFE SET as select statement.
l.result = base.Select
}
case ctx.ShowStatement() != nil:
@@ -68,7 +81,6 @@ func (l *queryTypeListener) EnterSimpleStatement(ctx *mysql.SimpleStatementConte
}
}
default:
- // For any other statement type, default to Select
l.result = base.Select
}
}
@@ -78,9 +90,6 @@ func (l *queryTypeListener) EnterSelectStatement(ctx *mysql.SelectStatementConte
return
}
- // MySQL cannot use SELECT ... INTO .. FROM ... syntax to create a new table or insert into an existing table.
- // So we can safely assume it's a SELECT statement.
-
if l.allSystems {
l.result = base.SelectInfoSchema
} else {
diff --git a/backend/plugin/parser/mysql/restore.go b/backend/plugin/parser/mysql/restore.go
index 78d7a950c7..5217a6d484 100644
--- a/backend/plugin/parser/mysql/restore.go
+++ b/backend/plugin/parser/mysql/restore.go
@@ -5,10 +5,10 @@ import (
"fmt"
"strings"
- "github.com/antlr4-go/antlr/v4"
- parser "github.com/bytebase/parser/mysql"
"github.com/pkg/errors"
+ "github.com/bytebase/omni/mysql/ast"
+
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
@@ -28,28 +28,41 @@ func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statement
return "", errors.Errorf("failed to extract single SQL: %v", err)
}
- parseResult, err := ParseMySQL(originalSQL)
+ // Parse the filtered SQL and find the first DML node.
+ matchingNode, err := findFirstDML(originalSQL)
if err != nil {
return "", err
}
-
- if len(parseResult) == 0 {
- return "", errors.Errorf("no parse result")
+ if matchingNode == nil {
+ return "", errors.Errorf("no DML statement found in extracted SQL")
}
- // We only need the first parse result.
- // There are two cases:
- // 1. The statement is a single SQL statement.
- // 2. The statement is a multi SQL statement, but all SQL statements' backup is in the same table.
- // So we only need to restore the first SQL statement.
sqlForComment, truncated := common.TruncateString(originalSQL, maxCommentLength)
if truncated {
sqlForComment += "..."
}
- return doGenerate(ctx, rCtx, sqlForComment, parseResult[0], backupItem)
+ return doGenerate(ctx, rCtx, sqlForComment, matchingNode, backupItem)
}
-func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, ast *base.ANTLRAST, backupItem *storepb.PriorBackupDetail_Item) (string, error) {
+func findFirstDML(statement string) (ast.Node, error) {
+ stmtList, err := ParseMySQLOmni(statement)
+ if err != nil {
+ return nil, errors.Wrap(err, "failed to parse statement")
+ }
+ if stmtList == nil {
+ return nil, nil
+ }
+ for _, item := range stmtList.Items {
+ switch item.(type) {
+ case *ast.UpdateStmt, *ast.DeleteStmt:
+ return item, nil
+ default:
+ }
+ }
+ return nil, nil
+}
+
+func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, node ast.Node, backupItem *storepb.PriorBackupDetail_Item) (string, error) {
_, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)
if err != nil {
return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)
@@ -66,166 +79,183 @@ func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment str
return "", errors.Wrapf(err, "failed to classify columns for %s.%s", backupItem.SourceTable.Database, backupItem.SourceTable.Table)
}
- g := &generator{
- ctx: ctx,
- rCtx: rCtx,
- backupDatabase: targetDatabase,
- backupTable: backupItem.TargetTable.Table,
- originalDatabase: sourceDatabase,
- originalTable: backupItem.SourceTable.Table,
- generatedColumns: generatedColumns,
- normalColumns: normalColumns,
- }
- var buf strings.Builder
- antlr.ParseTreeWalkerDefault.Walk(g, ast.Tree)
- if g.err != nil {
- return "", g.err
- }
- if _, err := fmt.Fprintf(&buf, "/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, g.result); err != nil {
- return "", err
+ var result string
+ switch n := node.(type) {
+ case *ast.DeleteStmt:
+ result = generateDeleteRestore(sourceDatabase, backupItem.SourceTable.Table, targetDatabase, backupItem.TargetTable.Table, generatedColumns, normalColumns)
+ case *ast.UpdateStmt:
+ r, err := generateUpdateRestore(ctx, rCtx, n, sourceDatabase, backupItem.SourceTable.Table, targetDatabase, backupItem.TargetTable.Table, generatedColumns, normalColumns)
+ if err != nil {
+ return "", err
+ }
+ result = r
+ default:
+ return "", errors.Errorf("unexpected statement type: %T", node)
}
- return buf.String(), nil
+
+ return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, result), nil
}
-func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {
- if backupItem == nil {
- return "", errors.Errorf("backup item is nil")
+func generateDeleteRestore(originalDatabase, originalTable, backupDatabase, backupTable string, generatedColumns, normalColumns []string) string {
+ if len(generatedColumns) == 0 {
+ return fmt.Sprintf("INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s`;", originalDatabase, originalTable, backupDatabase, backupTable)
}
-
- list, err := SplitSQL(statement)
- if err != nil {
- return "", errors.Wrap(err, "failed to split sql")
+ var quotedColumns []string
+ for _, column := range normalColumns {
+ quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))
}
+ quotedColumnList := strings.Join(quotedColumns, ", ")
+ return fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s`;", originalDatabase, originalTable, quotedColumnList, quotedColumnList, backupDatabase, backupTable)
+}
- start := 0
- end := len(list) - 1
- for i, item := range list {
- if equalOrLess(item.Start, backupItem.StartPosition) {
- start = i
- }
- }
+func generateUpdateRestore(ctx context.Context, rCtx base.RestoreContext, stmt *ast.UpdateStmt, originalDatabase, originalTable, backupDatabase, backupTable string, generatedColumns, normalColumns []string) (string, error) {
+ // Extract single tables from the UPDATE table references.
+ singleTables := extractSingleTablesFromTableExprs(originalDatabase, stmt.Tables)
- for i := len(list) - 1; i >= 0; i-- {
- if equalOrGreater(list[i].Start, backupItem.EndPosition) {
- end = i
+ // Find the table matching the original table.
+ var matchedTable *TableReference
+ for _, table := range singleTables {
+ if strings.EqualFold(table.Table, originalTable) {
+ matchedTable = table
+ break
}
}
- _, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)
+ // Extract update column names that belong to the original table.
+ updateColumns := extractUpdateColumns(stmt.SetList, originalDatabase, originalTable, matchedTable, normalColumns)
+
+ has, err := hasDisjointUniqueKey(ctx, rCtx, originalDatabase, originalTable, updateColumns)
if err != nil {
- return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)
+ return "", err
+ }
+ if !has {
+ return "", errors.Errorf("no disjoint unique key found for %s.%s", originalDatabase, originalTable)
}
- var result []string
- // We only need statements that contain the source table.
- for i := start; i <= end; i++ {
- parseResult, err := ParseMySQL(list[i].Text)
- if err != nil {
- return "", errors.Wrap(err, "failed to parse sql")
+ var buf strings.Builder
+ if len(generatedColumns) == 0 {
+ if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", originalDatabase, originalTable, backupDatabase, backupTable); err != nil {
+ return "", err
}
- containsSourceTable := false
- for _, sql := range parseResult {
- tables, err := ExtractTables(sourceDatabase, sql, i, nil)
- if err != nil {
- return "", errors.Wrap(err, "failed to extract tables")
- }
- for _, table := range tables {
- if table.Table.Database == sourceDatabase && table.Table.Table == backupItem.SourceTable.Table {
- containsSourceTable = true
- break
- }
- }
- if containsSourceTable {
- break
- }
+ } else {
+ var quotedColumns []string
+ for _, column := range normalColumns {
+ quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))
}
- if containsSourceTable {
- result = append(result, list[i].Text)
+ quotedColumnList := strings.Join(quotedColumns, ", ")
+ if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", originalDatabase, originalTable, quotedColumnList, quotedColumnList, backupDatabase, backupTable); err != nil {
+ return "", err
}
}
- return strings.Join(result, ""), nil
-}
-func equalOrLess(a, b *storepb.Position) bool {
- if a.Line < b.Line {
- return true
+ for i, field := range updateColumns {
+ if i > 0 {
+ if _, err := buf.WriteString(", "); err != nil {
+ return "", err
+ }
+ }
+ if _, err := fmt.Fprintf(&buf, "`%s` = VALUES(`%s`)", field, field); err != nil {
+ return "", err
+ }
}
- if a.Line == b.Line && a.Column <= b.Column {
- return true
+ if _, err := buf.WriteString(";"); err != nil {
+ return "", err
}
- return false
+ return buf.String(), nil
}
-func equalOrGreater(a, b *storepb.Position) bool {
- if a.Line > b.Line {
- return true
- }
- if a.Line == b.Line && a.Column >= b.Column {
- return true
+// extractSingleTablesFromTableExprs walks omni TableExpr nodes and returns
+// a map of alias-or-name to TableReference for simple table references.
+func extractSingleTablesFromTableExprs(databaseName string, exprs []ast.TableExpr) map[string]*TableReference {
+ result := make(map[string]*TableReference)
+ for _, expr := range exprs {
+ collectTableRefs(databaseName, expr, result)
}
- return false
+ return result
}
-type generator struct {
- *parser.BaseMySQLParserListener
+func collectTableRefs(databaseName string, expr ast.TableExpr, result map[string]*TableReference) {
+ switch n := expr.(type) {
+ case *ast.TableRef:
+ database := n.Schema
+ if database == "" {
+ database = databaseName
+ }
+ table := &TableReference{
+ Database: database,
+ Table: n.Name,
+ Alias: n.Alias,
+ }
+ if n.Alias != "" {
+ result[n.Alias] = table
+ } else {
+ result[n.Name] = table
+ }
+ case *ast.JoinClause:
+ collectTableRefs(databaseName, n.Left, result)
+ collectTableRefs(databaseName, n.Right, result)
+ default:
+ }
+}
- ctx context.Context
- rCtx base.RestoreContext
+// extractUpdateColumns extracts column names from SET assignments that belong
+// to the original table.
+func extractUpdateColumns(setList []*ast.Assignment, database, _ string, matchedTable *TableReference, normalColumns []string) []string {
+ var result []string
+ for _, assignment := range setList {
+ col := assignment.Column
+ if col == nil {
+ continue
+ }
- backupDatabase string
- backupTable string
- originalDatabase string
- originalTable string
- generatedColumns []string
- normalColumns []string
- result string
- err error
-}
+ if col.Schema != "" && !strings.EqualFold(col.Schema, database) {
+ continue
+ }
-func (g *generator) EnterDeleteStatement(ctx *parser.DeleteStatementContext) {
- if !isTopLevel(ctx.GetParent()) {
- return
- }
+ if col.Table == "" {
+ // Unqualified column: check if it belongs to the normalColumns set.
+ for _, c := range normalColumns {
+ if strings.EqualFold(c, col.Column) {
+ result = append(result, col.Column)
+ break
+ }
+ }
+ continue
+ }
- if len(g.generatedColumns) == 0 {
- g.result = fmt.Sprintf("INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s`;", g.originalDatabase, g.originalTable, g.backupDatabase, g.backupTable)
- } else {
- var quotedColumns []string
- for _, column := range g.normalColumns {
- quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))
+ if matchedTable != nil && (col.Table == matchedTable.Alias || strings.EqualFold(col.Table, matchedTable.Table)) {
+ result = append(result, col.Column)
}
- quotedColumnList := strings.Join(quotedColumns, ", ")
- g.result = fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s`;", g.originalDatabase, g.originalTable, quotedColumnList, quotedColumnList, g.backupDatabase, g.backupTable)
}
+ return result
}
-func (g *generator) hasDisjointUniqueKey(updateColumns []string) (bool, error) {
+func hasDisjointUniqueKey(ctx context.Context, rCtx base.RestoreContext, originalDatabase, originalTable string, updateColumns []string) (bool, error) {
columnMap := make(map[string]bool)
for _, column := range updateColumns {
columnMap[strings.ToLower(column)] = true
}
- if g.rCtx.GetDatabaseMetadataFunc == nil {
+ if rCtx.GetDatabaseMetadataFunc == nil {
return false, errors.Errorf("GetDatabaseMetadataFunc is nil")
}
- _, metadata, err := g.rCtx.GetDatabaseMetadataFunc(g.ctx, g.rCtx.InstanceID, g.originalDatabase)
+ _, metadata, err := rCtx.GetDatabaseMetadataFunc(ctx, rCtx.InstanceID, originalDatabase)
if err != nil {
- return false, errors.Wrapf(err, "failed to get database metadata for %s", g.originalDatabase)
+ return false, errors.Wrapf(err, "failed to get database metadata for %s", originalDatabase)
}
-
if metadata == nil {
- return false, errors.Errorf("database metadata is nil for %s", g.originalDatabase)
+ return false, errors.Errorf("database metadata is nil for %s", originalDatabase)
}
schema := metadata.GetSchemaMetadata("")
if schema == nil {
- return false, errors.Errorf("schema is nil for %s", g.originalDatabase)
+ return false, errors.Errorf("schema is nil for %s", originalDatabase)
}
- tableMetadata := schema.GetTable(g.originalTable)
+ tableMetadata := schema.GetTable(originalTable)
if tableMetadata == nil {
- return false, errors.Errorf("table metadata is nil for %s.%s", g.originalDatabase, g.originalTable)
+ return false, errors.Errorf("table metadata is nil for %s.%s", originalDatabase, originalTable)
}
for _, index := range tableMetadata.GetProto().Indexes {
@@ -249,105 +279,110 @@ func disjoint(a []string, b map[string]bool) bool {
return true
}
-func (g *generator) EnterUpdateStatement(ctx *parser.UpdateStatementContext) {
- if !isTopLevel(ctx.GetParent()) {
- return
+func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {
+ if backupItem == nil {
+ return "", errors.Errorf("backup item is nil")
}
- singleTables := &singleTableListener{
- databaseName: g.originalDatabase,
- singleTables: make(map[string]*TableReference),
+ list, err := SplitSQL(statement)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to split sql")
}
- antlr.ParseTreeWalkerDefault.Walk(singleTables, ctx.TableReferenceList())
-
- updateItems := &updateItemListener{
- database: g.originalDatabase,
- normalColumns: g.normalColumns,
+ start := 0
+ end := len(list) - 1
+ for i, item := range list {
+ if equalOrLess(item.Start, backupItem.StartPosition) {
+ start = i
+ }
}
- for _, table := range singleTables.singleTables {
- if strings.EqualFold(table.Table, g.originalTable) {
- updateItems.table = table
- break
+
+ for i := len(list) - 1; i >= 0; i-- {
+ if equalOrGreater(list[i].Start, backupItem.EndPosition) {
+ end = i
}
}
- antlr.ParseTreeWalkerDefault.Walk(updateItems, ctx.UpdateList())
- has, err := g.hasDisjointUniqueKey(updateItems.result)
+ _, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)
if err != nil {
- g.err = err
- return
- }
- if !has {
- g.err = errors.Errorf("no disjoint unique key found for %s.%s", g.originalDatabase, g.originalTable)
- return
+ return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)
}
- var buf strings.Builder
- if len(g.generatedColumns) == 0 {
- if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", g.originalDatabase, g.originalTable, g.backupDatabase, g.backupTable); err != nil {
- g.err = err
- return
+ var result []string
+ for i := start; i <= end; i++ {
+ stmtList, err := ParseMySQLOmni(list[i].Text)
+ if err != nil {
+ return "", errors.Wrap(err, "failed to parse sql")
}
- } else {
- var quotedColumns []string
- for _, column := range g.normalColumns {
- quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))
+ containsSourceTable := false
+ for _, node := range stmtList.Items {
+ if containsTable(node, sourceDatabase, backupItem.SourceTable.Table) {
+ containsSourceTable = true
+ break
+ }
}
- quotedColumnList := strings.Join(quotedColumns, ", ")
- if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", g.originalDatabase, g.originalTable, quotedColumnList, quotedColumnList, g.backupDatabase, g.backupTable); err != nil {
- g.err = err
- return
+ if containsSourceTable {
+ result = append(result, list[i].Text)
}
}
+ return strings.Join(result, ""), nil
+}
- for i, field := range updateItems.result {
- if i > 0 {
- if _, err := buf.WriteString(", "); err != nil {
- g.err = err
- return
+// containsTable checks whether the given AST node references the specified table.
+func containsTable(node ast.Node, database, table string) bool {
+ switch n := node.(type) {
+ case *ast.UpdateStmt:
+ for _, expr := range n.Tables {
+ if tableExprReferences(expr, database, table) {
+ return true
}
}
-
- if _, err := fmt.Fprintf(&buf, "`%s` = VALUES(`%s`)", field, field); err != nil {
- g.err = err
- return
+ case *ast.DeleteStmt:
+ for _, expr := range n.Tables {
+ if tableExprReferences(expr, database, table) {
+ return true
+ }
}
+ for _, expr := range n.Using {
+ if tableExprReferences(expr, database, table) {
+ return true
+ }
+ }
+ default:
}
- if _, err := buf.WriteString(";"); err != nil {
- g.err = err
- return
- }
- g.result = buf.String()
+ return false
}
-type updateItemListener struct {
- *parser.BaseMySQLParserListener
- normalColumns []string
- database string
- table *TableReference
- result []string
+func tableExprReferences(expr ast.TableExpr, database, table string) bool {
+ switch n := expr.(type) {
+ case *ast.TableRef:
+ db := n.Schema
+ if db == "" {
+ db = database
+ }
+ return db == database && strings.EqualFold(n.Name, table)
+ case *ast.JoinClause:
+ return tableExprReferences(n.Left, database, table) || tableExprReferences(n.Right, database, table)
+ }
+ return false
}
-func (l *updateItemListener) EnterUpdateElement(ctx *parser.UpdateElementContext) {
- database, table, column := NormalizeMySQLColumnRef(ctx.ColumnRef())
-
- if database != "" && !strings.EqualFold(database, l.database) {
- return
+func equalOrLess(a, b *storepb.Position) bool {
+ if a.Line < b.Line {
+ return true
}
-
- if table == "" {
- for _, c := range l.normalColumns {
- if strings.EqualFold(c, column) {
- l.result = append(l.result, column)
- return
- }
- }
- return
+ if a.Line == b.Line && a.Column <= b.Column {
+ return true
}
+ return false
+}
- if l.table.Alias == table || strings.EqualFold(l.table.Table, table) {
- l.result = append(l.result, column)
- return
+func equalOrGreater(a, b *storepb.Position) bool {
+ if a.Line > b.Line {
+ return true
}
+ if a.Line == b.Line && a.Column >= b.Column {
+ return true
+ }
+ return false
}