Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser/mysql): migrate backup, restore to omni AST; delete generic_checker [BYT-9119, BYT-9120] (#19848)
rebelice7fbce68ef086
diff --git a/backend/plugin/advisor/mysql/generic_checker.go b/backend/plugin/advisor/mysql/generic_checker.godeleted file mode 100644index b9ab3f666d..0000000000--- a/backend/plugin/advisor/mysql/generic_checker.go+++ /dev/null@@ -1,173 +0,0 @@-package mysql--import (- "fmt"- "reflect"- "strings"-- "github.com/antlr4-go/antlr/v4"- "github.com/bytebase/parser/mysql"-- storepb "github.com/bytebase/bytebase/backend/generated-go/store"-)--// Node type constants for consistent node type checking-const (- NodeTypeCreateTable = "CreateTable"- NodeTypeAlterTable = "AlterTable"- NodeTypeAlterStatement = "AlterStatement"- NodeTypeDropTable = "DropTable"- NodeTypeRenameTableStatement = "RenameTableStatement"- NodeTypeSetStatement = "SetStatement"- NodeTypeCreateIndex = "CreateIndex"- NodeTypeDropIndex = "DropIndex"- NodeTypeInsertStatement = "InsertStatement"- NodeTypeUpdateStatement = "UpdateStatement"- NodeTypeDeleteStatement = "DeleteStatement"- NodeTypeSelectStatement = "SelectStatement"- NodeTypeCreateView = "CreateView"- NodeTypeDropView = "DropView"- NodeTypeCreateProcedure = "CreateProcedure"- NodeTypeDropProcedure = "DropProcedure"- NodeTypeCreateFunction = "CreateFunction"- NodeTypeDropFunction = "DropFunction"- NodeTypeCreateEvent = "CreateEvent"- NodeTypeDropEvent = "DropEvent"- NodeTypeCreateTrigger = "CreateTrigger"- NodeTypeDropTrigger = "DropTrigger"- NodeTypeQuery = "Query"- NodeTypeQueryExpression = "QueryExpression"- NodeTypeFunctionCall = "FunctionCall"- NodeTypeCreateDatabase = "CreateDatabase"- NodeTypeAlterDatabase = "AlterDatabase"- NodeTypeDropDatabase = "DropDatabase"- NodeTypeTruncateTableStatement = "TruncateTableStatement"- NodeTypeSelectStatementWithInto = "SelectStatementWithInto"- NodeTypeSelectItemList = "SelectItemList"- NodeTypePureIdentifier = "PureIdentifier"- NodeTypeIdentifierKeyword = "IdentifierKeyword"- NodeTypeTransactionStatement = "TransactionStatement"- NodeTypePredicateExprLike = "PredicateExprLike"- NodeTypeSimpleStatement = "SimpleStatement"- NodeTypeQuerySpecification = "QuerySpecification"- NodeTypeLimitClause = "LimitClause"- NodeTypeFromClause = "FromClause"- NodeTypePrimaryExprCompare = "PrimaryExprCompare"- NodeTypeJoinedTable = "JoinedTable"- NodeTypeWhereClause = "WhereClause"- NodeTypePredicateExprIn = "PredicateExprIn"- NodeTypeExprList = "ExprList"- NodeTypeExprOr = "ExprOr"- NodeTypeAlterTableActions = "AlterTableActions"-)--// Rule defines the interface for individual SQL validation rules.-// Each rule implements specific checking logic without embedding the base listener.-type Rule interface {- // OnEnter is called when entering a parse tree node- OnEnter(ctx antlr.ParserRuleContext, nodeType string) error-- // OnExit is called when exiting a parse tree node- OnExit(ctx antlr.ParserRuleContext, nodeType string) error-- // Name returns the rule name for logging/debugging- Name() string-- // GetAdviceList returns the accumulated advice from this rule- GetAdviceList() []*storepb.Advice-}--// GenericChecker embeds the base MySQL parser listener and dispatches events to registered rules.-// This design ensures only one copy of the listener type metadata in the binary.-type GenericChecker struct {- *mysql.BaseMySQLParserListener-- rules []Rule- baseLine int-}--// NewGenericChecker creates a new instance of GenericChecker with the given rules.-func NewGenericChecker(rules []Rule) *GenericChecker {- return &GenericChecker{- rules: rules,- }-}--// SetBaseLine sets the base line number for error reporting.-func (g *GenericChecker) SetBaseLine(baseLine int) {- g.baseLine = baseLine-}--// GetBaseLine returns the current base line number.-func (g *GenericChecker) GetBaseLine() int {- return g.baseLine-}--// EnterEveryRule is called when any rule is entered.-// It dispatches the event to all registered rules.-func (g *GenericChecker) EnterEveryRule(ctx antlr.ParserRuleContext) {- nodeType := g.getNodeType(ctx)- for _, rule := range g.rules {- if err := rule.OnEnter(ctx, nodeType); err != nil {- // Log error if needed- fmt.Printf("Rule %s error on enter %s: %v\n", rule.Name(), nodeType, err)- }- }-}--// ExitEveryRule is called when any rule is exited.-// It dispatches the event to all registered rules.-func (g *GenericChecker) ExitEveryRule(ctx antlr.ParserRuleContext) {- nodeType := g.getNodeType(ctx)- for _, rule := range g.rules {- if err := rule.OnExit(ctx, nodeType); err != nil {- // Log error if needed- fmt.Printf("Rule %s error on exit %s: %v\n", rule.Name(), nodeType, err)- }- }-}--// GetAdviceList collects and returns all advice from registered rules.-func (g *GenericChecker) GetAdviceList() []*storepb.Advice {- var allAdvice []*storepb.Advice- for _, rule := range g.rules {- allAdvice = append(allAdvice, rule.GetAdviceList()...)- }- return allAdvice-}--// getNodeType returns the type name of the parse tree node.-func (*GenericChecker) getNodeType(ctx antlr.ParserRuleContext) string {- t := reflect.TypeOf(ctx)- if t.Kind() == reflect.Ptr {- t = t.Elem()- }- name := t.Name()- // Remove "Context" suffix if present- name = strings.TrimSuffix(name, "Context")- return name-}--// BaseRule provides common functionality for rules.-// Other rules can embed this struct to get common behavior.-type BaseRule struct {- level storepb.Advice_Status // nolint:unused- title string // nolint:unused- adviceList []*storepb.Advice- baseLine int-}--// SetBaseLine sets the base line for the rule.-func (r *BaseRule) SetBaseLine(baseLine int) {- r.baseLine = baseLine-}--// GetAdviceList returns the accumulated advice.-func (r *BaseRule) GetAdviceList() []*storepb.Advice {- return r.adviceList-}--// AddAdvice adds a new advice to the list.-func (r *BaseRule) AddAdvice(advice *storepb.Advice) {- r.adviceList = append(r.adviceList, advice)-}diff --git a/backend/plugin/advisor/mysql/rule_builtin_prior_backup_check.go b/backend/plugin/advisor/mysql/rule_builtin_prior_backup_check.goindex 23d27037ea..3a0a8e3d3f 100644--- a/backend/plugin/advisor/mysql/rule_builtin_prior_backup_check.go+++ b/backend/plugin/advisor/mysql/rule_builtin_prior_backup_check.go@@ -117,11 +117,11 @@ func prepareTransformation(databaseName string, parsedStatements []base.ParsedStif stmt.AST == nil {continue}- ast, ok := base.GetANTLRAST(stmt.AST)+ node, ok := mysqlparser.GetOmniNode(stmt.AST)if !ok {continue}- tables, err := mysqlparser.ExtractTables(databaseName, ast, i, nil)+ tables, err := mysqlparser.ExtractTables(databaseName, node, stmt.Text, i, nil)if err != nil {return nil, errors.Wrap(err, "failed to extract tables")}diff --git a/backend/plugin/parser/mysql/backup.go b/backend/plugin/parser/mysql/backup.goindex b27c6ef3ac..5fa119d984 100644--- a/backend/plugin/parser/mysql/backup.go+++ b/backend/plugin/parser/mysql/backup.go@@ -7,42 +7,24 @@ import ("slices""strings"- "github.com/antlr4-go/antlr/v4""github.com/pkg/errors"- parser "github.com/bytebase/parser/mysql"+ "github.com/bytebase/omni/mysql/ast""github.com/bytebase/bytebase/backend/common"- "github.com/bytebase/bytebase/backend/generated-go/store"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store/model")func init() {- base.RegisterTransformDMLToSelect(store.Engine_MYSQL, TransformDMLToSelect)+ base.RegisterTransformDMLToSelect(storepb.Engine_MYSQL, TransformDMLToSelect)}const (maxTableNameLength = 64)-func TransformDMLToSelect(ctx context.Context, tCtx base.TransformContext, statement string, sourceDatabase string, targetDatabase string, tablePrefix string) ([]base.BackupStatement, error) {- var dbMetadata *model.DatabaseMetadata- if tCtx.GetDatabaseMetadataFunc != nil {- _, metadata, err := tCtx.GetDatabaseMetadataFunc(ctx, tCtx.InstanceID, sourceDatabase)- if err == nil {- dbMetadata = metadata- }- }-- statementInfoList, err := prepareTransformation(sourceDatabase, statement, dbMetadata)- if err != nil {- return nil, errors.Wrap(err, "failed to prepare transformation")- }-- return generateSQL(ctx, tCtx, statementInfoList, targetDatabase, tablePrefix)-}-type StatementType intconst (@@ -62,10 +44,28 @@ type TableReference struct {type StatementInfo struct {Offset intStatement string- Tree antlr.ParserRuleContext+ Node ast.NodeTable *TableReference- StartPosition *store.Position- EndPosition *store.Position+ StartPosition *storepb.Position+ EndPosition *storepb.Position+ FullSQL string+}++func TransformDMLToSelect(ctx context.Context, tCtx base.TransformContext, statement string, sourceDatabase string, targetDatabase string, tablePrefix string) ([]base.BackupStatement, error) {+ var dbMetadata *model.DatabaseMetadata+ if tCtx.GetDatabaseMetadataFunc != nil {+ _, metadata, err := tCtx.GetDatabaseMetadataFunc(ctx, tCtx.InstanceID, sourceDatabase)+ if err == nil {+ dbMetadata = metadata+ }+ }++ statementInfoList, err := prepareTransformation(sourceDatabase, statement, dbMetadata)+ if err != nil {+ return nil, errors.Wrap(err, "failed to prepare transformation")+ }++ return generateSQL(ctx, tCtx, statementInfoList, targetDatabase, tablePrefix)}func prepareTransformation(databaseName, statement string, dbMetadata *model.DatabaseMetadata) ([]StatementInfo, error) {@@ -75,32 +75,33 @@ func prepareTransformation(databaseName, statement string, dbMetadata *model.Dat}var result []StatementInfo-for i, item := range list {if len(item.Text) == 0 || item.Empty {continue}- parseResult, err := ParseMySQL(item.Text)++ parsed, err := ParseMySQLOmni(item.Text)if err != nil {return nil, errors.Wrap(err, "failed to parse sql")}+ if parsed == nil || len(parsed.Items) == 0 {+ continue+ }- for _, sql := range parseResult {- // After splitting the SQL, we should have only one statement in the list.- // The FOR loop is just for safety.- // So we can use the i as the offset.- tables, err := ExtractTables(databaseName, sql, i, dbMetadata)+ for _, node := range parsed.Items {+ tables, err := ExtractTables(databaseName, node, item.Text, i, dbMetadata)if err != nil {return nil, errors.Wrap(err, "failed to extract tables")}for _, table := range tables {result = append(result, StatementInfo{- Offset: i,+ Offset: table.Offset,Statement: table.Statement,+ Node: table.Node,Table: table.Table,- Tree: table.Tree,StartPosition: item.Start,EndPosition: item.End,+ FullSQL: item.Text,})}}@@ -109,6 +110,367 @@ func prepareTransformation(databaseName, statement string, dbMetadata *model.Datreturn result, nil}+// ExtractTables extracts table references from a DML statement.+func ExtractTables(databaseName string, node ast.Node, fullSQL string, offset int, dbMetadata *model.DatabaseMetadata) ([]StatementInfo, error) {+ switch n := node.(type) {+ case *ast.DeleteStmt:+ return extractTablesFromDelete(databaseName, n, fullSQL, offset, dbMetadata)+ case *ast.UpdateStmt:+ return extractTablesFromUpdate(databaseName, n, fullSQL, offset, dbMetadata)+ default:+ return nil, nil+ }+}++func extractTablesFromDelete(databaseName string, n *ast.DeleteStmt, fullSQL string, offset int, _ *model.DatabaseMetadata) ([]StatementInfo, error) {+ cteNames := collectCTENames(fullSQL, n.Loc)+ stmtText := extractStatementText(fullSQL, n.Loc)++ // Collect all single table refs from the Tables list.+ singleTables := collectSingleTables(databaseName, n.Tables)++ if len(n.Using) == 0 {+ // Single-table DELETE: DELETE FROM t WHERE ...+ if len(singleTables) == 0 {+ return nil, nil+ }+ for _, ref := range singleTables {+ ref.StatementType = StatementTypeDelete+ }+ first := singleTables[firstKey(singleTables)]+ return []StatementInfo{{+ Offset: offset,+ Statement: stmtText,+ Node: n,+ Table: first,+ FullSQL: fullSQL,+ }}, nil+ }++ // Multi-table DELETE: DELETE t1, t2 FROM t1 JOIN t2 ... WHERE ...+ // Tables = targets to delete from, Using = referenced table list.+ refTables := collectSingleTables(databaseName, n.Using)++ var result []StatementInfo+ for _, target := range singleTables {+ name := target.Table+ if target.Alias != "" {+ name = target.Alias+ }+ if cteNames[name] {+ continue+ }+ ref, ok := refTables[name]+ if !ok {+ return nil, errors.Errorf("cannot extract reference table: no matched table %q in referenced table list", name)+ }+ ref.StatementType = StatementTypeDelete+ result = append(result, StatementInfo{+ Offset: offset,+ Statement: stmtText,+ Node: n,+ Table: ref,+ FullSQL: fullSQL,+ })+ }+ return result, nil+}++func extractTablesFromUpdate(databaseName string, n *ast.UpdateStmt, fullSQL string, offset int, dbMetadata *model.DatabaseMetadata) ([]StatementInfo, error) {+ cteNames := collectCTENames(fullSQL, n.Loc)+ stmtText := extractStatementText(fullSQL, n.Loc)+ singleTables := collectSingleTables(databaseName, n.Tables)++ // Determine which tables are updated via SET clause column prefixes.+ updatedTables := make(map[string]bool)+ var unqualifiedColumns []string+ for _, assign := range n.SetList {+ if assign.Column != nil {+ table := assign.Column.Table+ if !cteNames[table] {+ updatedTables[table] = true+ if table == "" {+ unqualifiedColumns = append(unqualifiedColumns, assign.Column.Column)+ }+ }+ }+ }++ // Resolve unqualified columns using metadata.+ if updatedTables[""] {+ delete(updatedTables, "")+ resolved := resolveUnqualifiedColumns(unqualifiedColumns, singleTables, dbMetadata)+ for t := range resolved {+ updatedTables[t] = true+ }+ }++ // Single-table UPDATE (only one table, no explicit qualification needed).+ if len(updatedTables) == 0 && len(singleTables) == 1 {+ for _, ref := range singleTables {+ ref.StatementType = StatementTypeUpdate+ return []StatementInfo{{+ Offset: offset,+ Statement: stmtText,+ Node: n,+ Table: ref,+ FullSQL: fullSQL,+ }}, nil+ }+ }++ var result []StatementInfo+ for table := range updatedTables {+ ref, ok := singleTables[table]+ if !ok {+ return nil, errors.Errorf("cannot extract reference table: no matched updated table %q in referenced table list", table)+ }+ ref.StatementType = StatementTypeUpdate+ result = append(result, StatementInfo{+ Offset: offset,+ Statement: stmtText,+ Node: n,+ Table: ref,+ FullSQL: fullSQL,+ })+ }+ return result, nil+}++// collectSingleTables walks TableExpr slices and collects TableRef entries keyed by alias or name.+func collectSingleTables(databaseName string, exprs []ast.TableExpr) map[string]*TableReference {+ result := make(map[string]*TableReference)+ for _, expr := range exprs {+ collectSingleTablesFromExpr(databaseName, expr, result)+ }+ return result+}++func collectSingleTablesFromExpr(databaseName string, expr ast.TableExpr, out map[string]*TableReference) {+ switch e := expr.(type) {+ case *ast.TableRef:+ db := e.Schema+ if db == "" {+ db = databaseName+ }+ ref := &TableReference{+ Database: db,+ Table: e.Name,+ Alias: e.Alias,+ }+ key := e.Name+ if e.Alias != "" {+ key = e.Alias+ }+ out[key] = ref+ case *ast.JoinClause:+ if e.Left != nil {+ collectSingleTablesFromExpr(databaseName, e.Left, out)+ }+ if e.Right != nil {+ collectSingleTablesFromExpr(databaseName, e.Right, out)+ }+ default:+ }+}++// collectCTENames extracts CTE names from the SQL text before the statement Loc.+// MySQL's UPDATE/DELETE CTEs are not in the omni AST, so we parse the prefix.+func collectCTENames(fullSQL string, stmtLoc ast.Loc) map[string]bool {+ result := make(map[string]bool)+ for _, cte := range parseCTEPrefix(fullSQL, stmtLoc) {+ result[cte.Name] = true+ }+ return result+}++// extractCTE extracts the CTE (WITH clause) text from the SQL before the statement.+func extractCTE(fullSQL string, stmtLoc ast.Loc) string {+ if stmtLoc.Start <= 0 {+ return ""+ }+ ctes := parseCTEPrefix(fullSQL, stmtLoc)+ if len(ctes) == 0 {+ return ""+ }+ return strings.TrimSpace(fullSQL[:stmtLoc.Start])+}++// parseCTEPrefix parses the SQL text before stmtLoc as a WITH clause.+// Returns the CTE list if the prefix contains a valid WITH clause, nil otherwise.+func parseCTEPrefix(fullSQL string, stmtLoc ast.Loc) []*ast.CommonTableExpr {+ if stmtLoc.Start <= 0 {+ return nil+ }+ prefix := strings.TrimSpace(fullSQL[:stmtLoc.Start])+ if prefix == "" {+ return nil+ }+ // Wrap the prefix with a dummy SELECT to make it parseable.+ parsed, err := ParseMySQLOmni(prefix + " SELECT 1")+ if err != nil || parsed == nil {+ return nil+ }+ for _, item := range parsed.Items {+ if sel, ok := item.(*ast.SelectStmt); ok && len(sel.CTEs) > 0 {+ return sel.CTEs+ }+ }+ return nil+}++func extractStatementText(fullSQL string, loc ast.Loc) string {+ end := loc.End+ if end <= 0 || end > len(fullSQL) {+ end = len(fullSQL)+ }+ start := 0+ if loc.Start > 0 {+ start = loc.Start+ }+ return strings.TrimSpace(fullSQL[start:end])+}++// writeSuffixSelectClause writes the FROM ... WHERE ... suffix of a DML statement.+func writeSuffixSelectClause(buf *strings.Builder, node ast.Node, fullSQL string) error {+ switch n := node.(type) {+ case *ast.UpdateStmt:+ return writeUpdateSuffix(buf, n, fullSQL)+ case *ast.DeleteStmt:+ return writeDeleteSuffix(buf, n, fullSQL)+ }+ return nil+}++func writeUpdateSuffix(buf *strings.Builder, n *ast.UpdateStmt, sql string) error {+ // For UPDATE: write "table_refs WHERE ... ORDER BY ... LIMIT ..."+ // Use the last table's Loc.End as the boundary — everything from first table+ // to last table's end is the table reference list.+ if len(n.Tables) > 0 {+ start := nodeLocStart(n.Tables[0])+ end := nodeLocEnd(n.Tables[len(n.Tables)-1])+ if start >= 0 && end > start && end <= len(sql) {+ if _, err := buf.WriteString(strings.TrimSpace(sql[start:end])); err != nil {+ return err+ }+ }+ }++ // Everything after the last SET assignment is WHERE + ORDER BY + LIMIT.+ if len(n.SetList) > 0 {+ lastAssign := n.SetList[len(n.SetList)-1]+ afterSet := lastAssign.Loc.End+ stmtEnd := n.Loc.End+ if stmtEnd <= 0 || stmtEnd > len(sql) {+ stmtEnd = len(sql)+ }+ suffix := strings.TrimSpace(sql[afterSet:stmtEnd])+ if suffix != "" {+ if _, err := fmt.Fprintf(buf, " %s", suffix); err != nil {+ return err+ }+ }+ }++ return nil+}++func writeDeleteSuffix(buf *strings.Builder, n *ast.DeleteStmt, sql string) error {+ // For single-table DELETE: everything from the table ref to statement end.+ // For multi-table DELETE: everything from the USING/reference table list to end.+ if len(n.Using) > 0 {+ start := nodeLocStart(n.Using[0])+ end := n.Loc.End+ if end <= 0 || end > len(sql) {+ end = len(sql)+ }+ if start >= 0 && end > start {+ if _, err := buf.WriteString(strings.TrimSpace(sql[start:end])); err != nil {+ return err+ }+ }+ } else if len(n.Tables) > 0 {+ start := nodeLocStart(n.Tables[0])+ end := n.Loc.End+ if end <= 0 || end > len(sql) {+ end = len(sql)+ }+ if start >= 0 && end > start {+ if _, err := buf.WriteString(strings.TrimSpace(sql[start:end])); err != nil {+ return err+ }+ }+ }+ return nil+}++func nodeLocStart(expr ast.TableExpr) int {+ switch e := expr.(type) {+ case *ast.TableRef:+ return e.Loc.Start+ case *ast.JoinClause:+ return e.Loc.Start+ }+ return -1+}++func nodeLocEnd(expr ast.TableExpr) int {+ switch e := expr.(type) {+ case *ast.TableRef:+ return e.Loc.End+ case *ast.JoinClause:+ return e.Loc.End+ }+ return -1+}++// resolveUnqualifiedColumns resolves unqualified column names to owning tables using metadata.+func resolveUnqualifiedColumns(columns []string, singleTables map[string]*TableReference, dbMetadata *model.DatabaseMetadata) map[string]bool {+ result := make(map[string]bool)++ if dbMetadata == nil {+ for t := range singleTables {+ result[t] = true+ }+ return result+ }+ schema := dbMetadata.GetSchemaMetadata("")+ if schema == nil {+ for t := range singleTables {+ result[t] = true+ }+ return result+ }++ for _, col := range columns {+ resolved := false+ for aliasOrName, ref := range singleTables {+ tableMetadata := schema.GetTable(ref.Table)+ if tableMetadata == nil {+ continue+ }+ if tableMetadata.GetColumn(col) != nil {+ result[aliasOrName] = true+ resolved = true+ }+ }+ if !resolved {+ for t := range singleTables {+ result[t] = true+ }+ return result+ }+ }+ return result+}++func firstKey(m map[string]*TableReference) string {+ for k := range m {+ return k+ }+ return ""+}+func generateSQL(ctx context.Context, tCtx base.TransformContext, statementInfoList []StatementInfo, databaseName string, tablePrefix string) ([]base.BackupStatement, error) {groupByTable := make(map[string][]StatementInfo)for _, item := range statementInfoList {@@ -116,7 +478,6 @@ func generateSQL(ctx context.Context, tCtx base.TransformContext, statementInfoLgroupByTable[key] = append(groupByTable[key], item)}- // Check if the statement type is the same for all statements in one table.for key, list := range groupByTable {stmtType := StatementTypeUnknownfor _, item := range list {@@ -130,7 +491,6 @@ func generateSQL(ctx context.Context, tCtx base.TransformContext, statementInfoL}var result []base.BackupStatement-for key, list := range groupByTable {backupStatement, err := generateSQLForTable(ctx, tCtx, list, databaseName, tablePrefix)if err != nil {@@ -202,8 +562,6 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statem}for i, item := range statementInfoList {if i != 0 {- // We assume that the source table has a primary key.- // If we have multiple statements, we need to use UNION DISTINCT to avoid duplicate rows.if _, err := buf.WriteString("\n UNION DISTINCT\n"); err != nil {return nil, errors.Wrap(err, "failed to write union all statement")}@@ -215,12 +573,15 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statemif _, err := buf.WriteString(" "); err != nil {return nil, errors.Wrap(err, "failed to write space")}- cteString := extractCTE(item.Tree)++ stmtLoc := nodeStmtLoc(item.Node)+ cteString := extractCTE(item.FullSQL, stmtLoc)if len(cteString) > 0 {if _, err := fmt.Fprintf(&buf, "%s ", cteString); err != nil {return nil, errors.Wrap(err, "failed to write cte")}}+if len(generatedColumns) == 0 {if _, err := fmt.Fprintf(&buf, "SELECT `%s`.* FROM ", tableNameOrAlias); err != nil {return nil, errors.Wrap(err, "failed to write select statement")@@ -229,8 +590,8 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statemif _, err := buf.WriteString("SELECT "); err != nil {return nil, errors.Wrap(err, "failed to write select statement")}- for i, column := range normalColumns {- if i > 0 {+ for j, column := range normalColumns {+ if j > 0 {if err := buf.WriteByte(','); err != nil {return nil, errors.Wrap(err, "failed to write comma")}@@ -243,7 +604,7 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statemreturn nil, errors.Wrap(err, "failed to write from")}}- if err := extractSuffixSelectStatement(item.Tree, &buf); err != nil {+ if err := writeSuffixSelectClause(&buf, item.Node, item.FullSQL); err != nil {return nil, errors.Wrap(err, "failed to extract suffix select statement")}}@@ -261,20 +622,14 @@ func generateSQLForTable(ctx context.Context, tCtx base.TransformContext, statem}, nil}-func extractCTE(ctx antlr.ParserRuleContext) string {- switch node := ctx.(type) {- case *parser.UpdateStatementContext:- if node.WithClause() != nil {- return node.GetParser().GetTokenStream().GetTextFromRuleContext(node.WithClause())- }- case *parser.DeleteStatementContext:- if node.WithClause() != nil {- return node.GetParser().GetTokenStream().GetTextFromRuleContext(node.WithClause())- }- default:+func nodeStmtLoc(node ast.Node) ast.Loc {+ switch n := node.(type) {+ case *ast.UpdateStmt:+ return n.Loc+ case *ast.DeleteStmt:+ return n.Loc}-- return ""+ return ast.Loc{}}func classifyColumns(ctx context.Context, getDatabaseMetadataFunc base.GetDatabaseMetadataFunc, listDatabaseNamesFunc base.ListDatabaseNamesFunc, isCaseSensitive bool, instanceID string, table *TableReference) ([]string, []string, error) {@@ -313,8 +668,7 @@ func classifyColumns(ctx context.Context, getDatabaseMetadataFunc base.GetDatabareturn nil, nil, errors.Errorf("failed to get database metadata for InstanceID %q, Database %q", instanceID, table.Database)}- emptySchema := ""- schema := dbMetadata.GetSchemaMetadata(emptySchema)+ schema := dbMetadata.GetSchemaMetadata("")if schema == nil {return nil, nil, errors.New("failed to get schema metadata")}@@ -345,359 +699,3 @@ func classifyColumns(ctx context.Context, getDatabaseMetadataFunc base.GetDatabareturn generatedColumns, normalColumns, nil}--func extractSuffixSelectStatement(tree antlr.Tree, buf *strings.Builder) error {- listener := &suffixSelectStatementListener{- buf: buf,- }-- antlr.ParseTreeWalkerDefault.Walk(listener, tree)- return listener.err-}--type suffixSelectStatementListener struct {- *parser.BaseMySQLParserListener-- buf *strings.Builder- err error-}--func (l *suffixSelectStatementListener) EnterDeleteStatement(ctx *parser.DeleteStatementContext) {- if !isTopLevel(ctx.GetParent()) {- return- }- if ctx.TableRef() != nil {- // Single table delete statement.- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromTokens(- ctx.TableRef().GetStart(),- ctx.GetStop(),- )); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }- }-- if ctx.TableAliasRefList() != nil {- // Multi table delete statement.- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromTokens(- ctx.TableReferenceList().GetStart(),- ctx.GetStop(),- )); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }- }-}--func (l *suffixSelectStatementListener) EnterUpdateStatement(ctx *parser.UpdateStatementContext) {- if !isTopLevel(ctx.GetParent()) {- return- }- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.TableReferenceList())); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }-- if ctx.WhereClause() != nil {- if err := l.buf.WriteByte(' '); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.WhereClause())); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }- }-- if ctx.OrderClause() != nil {- if err := l.buf.WriteByte(' '); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.OrderClause())); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }- }-- if ctx.SimpleLimitClause() != nil {- if err := l.buf.WriteByte(' '); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }- if _, err := l.buf.WriteString(ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx.SimpleLimitClause())); err != nil {- l.err = errors.Wrap(err, "failed to write suffix select statement")- return- }- }-}--func ExtractTables(databaseName string, ast *base.ANTLRAST, offset int, dbMetadata *model.DatabaseMetadata) ([]StatementInfo, error) {- listener := &tableReferenceListener{- databaseName: databaseName,- dbMetadata: dbMetadata,- offset: offset,- }-- antlr.ParseTreeWalkerDefault.Walk(listener, ast.Tree)-- return listener.tables, listener.err-}--type tableReferenceListener struct {- *parser.BaseMySQLParserListener-- databaseName string- dbMetadata *model.DatabaseMetadata- offset int- tables []StatementInfo- err error-}--func isTopLevel(ctx antlr.Tree) bool {- if ctx == nil {- return true- }- switch ctx := ctx.(type) {- case *parser.SimpleStatementContext:- return isTopLevel(ctx.GetParent())- case *parser.QueryContext, *parser.ScriptContext:- return true- default:- return false- }-}--func (l *tableReferenceListener) EnterDeleteStatement(ctx *parser.DeleteStatementContext) {- if !isTopLevel(ctx.GetParent()) {- return- }-- cteMap := make(map[string]bool)- if ctx.WithClause() != nil {- for _, cte := range ctx.WithClause().AllCommonTableExpression() {- tableName := NormalizeMySQLIdentifier(cte.Identifier())- cteMap[tableName] = true- }- }-- if ctx.TableRef() != nil {- // Single table delete statement.- database, table := NormalizeMySQLTableRef(ctx.TableRef())- if len(database) > 0 && database != l.databaseName {- l.err = errors.Errorf("database is not matched: %s != %s", database, l.databaseName)- return- }-- alias := ""-- if ctx.TableAlias() != nil {- alias = NormalizeMySQLIdentifier(ctx.TableAlias().Identifier())- }-- if len(database) == 0 {- database = l.databaseName- }-- l.tables = append(l.tables, StatementInfo{- Offset: l.offset,- Statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),- Tree: ctx,- Table: &TableReference{- Database: database,- Table: table,- Alias: alias,- StatementType: StatementTypeDelete,- },- })- return- }-- if ctx.TableAliasRefList() != nil {- // Multi table delete statement.- singleTables := &singleTableListener{- databaseName: l.databaseName,- singleTables: make(map[string]*TableReference),- }-- antlr.ParseTreeWalkerDefault.Walk(singleTables, ctx.TableReferenceList())-- for _, tableRef := range ctx.TableAliasRefList().AllTableRefWithWildcard() {- database, table := NormalizeMySQLTableRefWithWildcard(tableRef)- if len(database) > 0 && database != l.databaseName {- l.err = errors.Errorf("database is not matched: %s != %s", database, l.databaseName)- return- }-- if len(database) == 0 && cteMap[table] {- continue- }-- singleTable, ok := singleTables.singleTables[table]- if !ok {- l.err = errors.Errorf("cannot extract reference table: no matched table %q in referenced table list", table)- return- }-- singleTable.StatementType = StatementTypeDelete- l.tables = append(l.tables, StatementInfo{- Offset: l.offset,- Statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),- Tree: ctx,- Table: singleTable,- })- }- }-}--// resolveUnqualifiedColumns resolves unqualified column names to their owning-// table(s) using database metadata. Falls back to all tables if metadata is-// unavailable or column cannot be resolved.-func (l *tableReferenceListener) resolveUnqualifiedColumns(columns []string, singleTables map[string]*TableReference) map[string]bool {- result := make(map[string]bool)-- schema := l.getSchemaMetadata()- if schema == nil {- // No metadata available, fall back to all tables.- for tableName := range singleTables {- result[tableName] = true- }- return result- }-- for _, col := range columns {- resolved := false- for aliasOrName, ref := range singleTables {- tableMetadata := schema.GetTable(ref.Table)- if tableMetadata == nil {- continue- }- if tableMetadata.GetColumn(col) != nil {- result[aliasOrName] = true- resolved = true- }- }- if !resolved {- // Column not found in any table metadata, fall back to all tables.- for tableName := range singleTables {- result[tableName] = true- }- return result- }- }-- return result-}--func (l *tableReferenceListener) getSchemaMetadata() *model.SchemaMetadata {- if l.dbMetadata == nil {- return nil- }- return l.dbMetadata.GetSchemaMetadata("")-}--func (l *tableReferenceListener) EnterUpdateStatement(ctx *parser.UpdateStatementContext) {- if !isTopLevel(ctx.GetParent()) {- return- }-- cteMap := make(map[string]bool)-- if ctx.WithClause() != nil {- for _, cte := range ctx.WithClause().AllCommonTableExpression() {- tableName := NormalizeMySQLIdentifier(cte.Identifier())- cteMap[tableName] = true- }- }-- listener := &updateTableListener{- tables: make(map[string]bool),- cteMap: cteMap,- }-- antlr.ParseTreeWalkerDefault.Walk(listener, ctx.UpdateList())-- singleTables := &singleTableListener{- databaseName: l.databaseName,- singleTables: make(map[string]*TableReference),- }-- antlr.ParseTreeWalkerDefault.Walk(singleTables, ctx.TableReferenceList())-- // When SET clause columns are unqualified (no table prefix), resolve them- // to the owning table using database metadata. MySQL allows unqualified- // columns when they are unambiguous.- if _, exists := listener.tables[""]; exists {- delete(listener.tables, "")- resolved := l.resolveUnqualifiedColumns(listener.unqualifiedColumns, singleTables.singleTables)- for tableName := range resolved {- listener.tables[tableName] = true- }- }-- for table := range listener.tables {- singleTable, ok := singleTables.singleTables[table]- if !ok {- l.err = errors.Errorf("cannot extract reference table: no matched updated table %q in referenced table list", table)- return- }-- singleTable.StatementType = StatementTypeUpdate- l.tables = append(l.tables, StatementInfo{- Offset: l.offset,- Statement: ctx.GetParser().GetTokenStream().GetTextFromRuleContext(ctx),- Tree: ctx,- Table: singleTable,- })- }-}--type singleTableListener struct {- *parser.BaseMySQLParserListener-- databaseName string- singleTables map[string]*TableReference- err error-}--func (l *singleTableListener) EnterSingleTable(ctx *parser.SingleTableContext) {- if l.err != nil {- return- }- database, tableName := NormalizeMySQLTableRef(ctx.TableRef())- if len(database) > 0 && database != l.databaseName {- l.err = errors.Errorf("database is not matched: %s != %s", database, l.databaseName)- }- if len(database) == 0 {- database = l.databaseName- }- table := &TableReference{- Database: database,- Table: tableName,- }-- if ctx.TableAlias() != nil {- table.Alias = NormalizeMySQLIdentifier(ctx.TableAlias().Identifier())- l.singleTables[table.Alias] = table- } else {- l.singleTables[table.Table] = table- }-}--type updateTableListener struct {- *parser.BaseMySQLParserListener-- tables map[string]bool- unqualifiedColumns []string- cteMap map[string]bool-}--func (l *updateTableListener) EnterUpdateElement(ctx *parser.UpdateElementContext) {- _, table, column := NormalizeMySQLColumnRef(ctx.ColumnRef())- if _, exists := l.cteMap[table]; !exists {- l.tables[table] = true- if table == "" {- l.unqualifiedColumns = append(l.unqualifiedColumns, column)- }- }-}diff --git a/backend/plugin/parser/mysql/query_type.go b/backend/plugin/parser/mysql/query_type.goindex be421d754d..5b0b86e55c 100644--- a/backend/plugin/parser/mysql/query_type.go+++ b/backend/plugin/parser/mysql/query_type.go@@ -1,17 +1,31 @@package mysqlimport (+ "github.com/antlr4-go/antlr/v4""github.com/bytebase/parser/mysql""github.com/bytebase/bytebase/backend/plugin/parser/base")+func isTopLevel(ctx antlr.Tree) bool {+ if ctx == nil {+ return true+ }+ switch ctx := ctx.(type) {+ case *mysql.SimpleStatementContext:+ return isTopLevel(ctx.GetParent())+ case *mysql.QueryContext, *mysql.ScriptContext:+ return true+ default:+ return false+ }+}+type queryTypeListener struct {*mysql.BaseMySQLParserListener- allSystems bool- result base.QueryType- // This is for skipping the query span.+ allSystems bool+ result base.QueryTypeisExplainAnalyze bool}@@ -42,7 +56,6 @@ func (l *queryTypeListener) EnterSimpleStatement(ctx *mysql.SimpleStatementContel.result = base.DMLcase ctx.SetStatement() != nil:if len(ctx.SetStatement().StartOptionValueList().AllPASSWORD_SYMBOL()) == 0 {- // Treat SAFE SET as select statement.l.result = base.Select}case ctx.ShowStatement() != nil:@@ -68,7 +81,6 @@ func (l *queryTypeListener) EnterSimpleStatement(ctx *mysql.SimpleStatementConte}}default:- // For any other statement type, default to Selectl.result = base.Select}}@@ -78,9 +90,6 @@ func (l *queryTypeListener) EnterSelectStatement(ctx *mysql.SelectStatementContereturn}- // MySQL cannot use SELECT ... INTO .. FROM ... syntax to create a new table or insert into an existing table.- // So we can safely assume it's a SELECT statement.-if l.allSystems {l.result = base.SelectInfoSchema} else {diff --git a/backend/plugin/parser/mysql/restore.go b/backend/plugin/parser/mysql/restore.goindex 78d7a950c7..5217a6d484 100644--- a/backend/plugin/parser/mysql/restore.go+++ b/backend/plugin/parser/mysql/restore.go@@ -5,10 +5,10 @@ import ("fmt""strings"- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/mysql""github.com/pkg/errors"+ "github.com/bytebase/omni/mysql/ast"+"github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"@@ -28,28 +28,41 @@ func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statementreturn "", errors.Errorf("failed to extract single SQL: %v", err)}- parseResult, err := ParseMySQL(originalSQL)+ // Parse the filtered SQL and find the first DML node.+ matchingNode, err := findFirstDML(originalSQL)if err != nil {return "", err}-- if len(parseResult) == 0 {- return "", errors.Errorf("no parse result")+ if matchingNode == nil {+ return "", errors.Errorf("no DML statement found in extracted SQL")}- // We only need the first parse result.- // There are two cases:- // 1. The statement is a single SQL statement.- // 2. The statement is a multi SQL statement, but all SQL statements' backup is in the same table.- // So we only need to restore the first SQL statement.sqlForComment, truncated := common.TruncateString(originalSQL, maxCommentLength)if truncated {sqlForComment += "..."}- return doGenerate(ctx, rCtx, sqlForComment, parseResult[0], backupItem)+ return doGenerate(ctx, rCtx, sqlForComment, matchingNode, backupItem)}-func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, ast *base.ANTLRAST, backupItem *storepb.PriorBackupDetail_Item) (string, error) {+func findFirstDML(statement string) (ast.Node, error) {+ stmtList, err := ParseMySQLOmni(statement)+ if err != nil {+ return nil, errors.Wrap(err, "failed to parse statement")+ }+ if stmtList == nil {+ return nil, nil+ }+ for _, item := range stmtList.Items {+ switch item.(type) {+ case *ast.UpdateStmt, *ast.DeleteStmt:+ return item, nil+ default:+ }+ }+ return nil, nil+}++func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, node ast.Node, backupItem *storepb.PriorBackupDetail_Item) (string, error) {_, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)if err != nil {return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)@@ -66,166 +79,183 @@ func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment strreturn "", errors.Wrapf(err, "failed to classify columns for %s.%s", backupItem.SourceTable.Database, backupItem.SourceTable.Table)}- g := &generator{- ctx: ctx,- rCtx: rCtx,- backupDatabase: targetDatabase,- backupTable: backupItem.TargetTable.Table,- originalDatabase: sourceDatabase,- originalTable: backupItem.SourceTable.Table,- generatedColumns: generatedColumns,- normalColumns: normalColumns,- }- var buf strings.Builder- antlr.ParseTreeWalkerDefault.Walk(g, ast.Tree)- if g.err != nil {- return "", g.err- }- if _, err := fmt.Fprintf(&buf, "/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, g.result); err != nil {- return "", err+ var result string+ switch n := node.(type) {+ case *ast.DeleteStmt:+ result = generateDeleteRestore(sourceDatabase, backupItem.SourceTable.Table, targetDatabase, backupItem.TargetTable.Table, generatedColumns, normalColumns)+ case *ast.UpdateStmt:+ r, err := generateUpdateRestore(ctx, rCtx, n, sourceDatabase, backupItem.SourceTable.Table, targetDatabase, backupItem.TargetTable.Table, generatedColumns, normalColumns)+ if err != nil {+ return "", err+ }+ result = r+ default:+ return "", errors.Errorf("unexpected statement type: %T", node)}- return buf.String(), nil++ return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, result), nil}-func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {- if backupItem == nil {- return "", errors.Errorf("backup item is nil")+func generateDeleteRestore(originalDatabase, originalTable, backupDatabase, backupTable string, generatedColumns, normalColumns []string) string {+ if len(generatedColumns) == 0 {+ return fmt.Sprintf("INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s`;", originalDatabase, originalTable, backupDatabase, backupTable)}-- list, err := SplitSQL(statement)- if err != nil {- return "", errors.Wrap(err, "failed to split sql")+ var quotedColumns []string+ for _, column := range normalColumns {+ quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))}+ quotedColumnList := strings.Join(quotedColumns, ", ")+ return fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s`;", originalDatabase, originalTable, quotedColumnList, quotedColumnList, backupDatabase, backupTable)+}- start := 0- end := len(list) - 1- for i, item := range list {- if equalOrLess(item.Start, backupItem.StartPosition) {- start = i- }- }+func generateUpdateRestore(ctx context.Context, rCtx base.RestoreContext, stmt *ast.UpdateStmt, originalDatabase, originalTable, backupDatabase, backupTable string, generatedColumns, normalColumns []string) (string, error) {+ // Extract single tables from the UPDATE table references.+ singleTables := extractSingleTablesFromTableExprs(originalDatabase, stmt.Tables)- for i := len(list) - 1; i >= 0; i-- {- if equalOrGreater(list[i].Start, backupItem.EndPosition) {- end = i+ // Find the table matching the original table.+ var matchedTable *TableReference+ for _, table := range singleTables {+ if strings.EqualFold(table.Table, originalTable) {+ matchedTable = table+ break}}- _, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)+ // Extract update column names that belong to the original table.+ updateColumns := extractUpdateColumns(stmt.SetList, originalDatabase, originalTable, matchedTable, normalColumns)++ has, err := hasDisjointUniqueKey(ctx, rCtx, originalDatabase, originalTable, updateColumns)if err != nil {- return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)+ return "", err+ }+ if !has {+ return "", errors.Errorf("no disjoint unique key found for %s.%s", originalDatabase, originalTable)}- var result []string- // We only need statements that contain the source table.- for i := start; i <= end; i++ {- parseResult, err := ParseMySQL(list[i].Text)- if err != nil {- return "", errors.Wrap(err, "failed to parse sql")+ var buf strings.Builder+ if len(generatedColumns) == 0 {+ if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", originalDatabase, originalTable, backupDatabase, backupTable); err != nil {+ return "", err}- containsSourceTable := false- for _, sql := range parseResult {- tables, err := ExtractTables(sourceDatabase, sql, i, nil)- if err != nil {- return "", errors.Wrap(err, "failed to extract tables")- }- for _, table := range tables {- if table.Table.Database == sourceDatabase && table.Table.Table == backupItem.SourceTable.Table {- containsSourceTable = true- break- }- }- if containsSourceTable {- break- }+ } else {+ var quotedColumns []string+ for _, column := range normalColumns {+ quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))}- if containsSourceTable {- result = append(result, list[i].Text)+ quotedColumnList := strings.Join(quotedColumns, ", ")+ if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", originalDatabase, originalTable, quotedColumnList, quotedColumnList, backupDatabase, backupTable); err != nil {+ return "", err}}- return strings.Join(result, ""), nil-}-func equalOrLess(a, b *storepb.Position) bool {- if a.Line < b.Line {- return true+ for i, field := range updateColumns {+ if i > 0 {+ if _, err := buf.WriteString(", "); err != nil {+ return "", err+ }+ }+ if _, err := fmt.Fprintf(&buf, "`%s` = VALUES(`%s`)", field, field); err != nil {+ return "", err+ }}- if a.Line == b.Line && a.Column <= b.Column {- return true+ if _, err := buf.WriteString(";"); err != nil {+ return "", err}- return false+ return buf.String(), nil}-func equalOrGreater(a, b *storepb.Position) bool {- if a.Line > b.Line {- return true- }- if a.Line == b.Line && a.Column >= b.Column {- return true+// extractSingleTablesFromTableExprs walks omni TableExpr nodes and returns+// a map of alias-or-name to TableReference for simple table references.+func extractSingleTablesFromTableExprs(databaseName string, exprs []ast.TableExpr) map[string]*TableReference {+ result := make(map[string]*TableReference)+ for _, expr := range exprs {+ collectTableRefs(databaseName, expr, result)}- return false+ return result}-type generator struct {- *parser.BaseMySQLParserListener+func collectTableRefs(databaseName string, expr ast.TableExpr, result map[string]*TableReference) {+ switch n := expr.(type) {+ case *ast.TableRef:+ database := n.Schema+ if database == "" {+ database = databaseName+ }+ table := &TableReference{+ Database: database,+ Table: n.Name,+ Alias: n.Alias,+ }+ if n.Alias != "" {+ result[n.Alias] = table+ } else {+ result[n.Name] = table+ }+ case *ast.JoinClause:+ collectTableRefs(databaseName, n.Left, result)+ collectTableRefs(databaseName, n.Right, result)+ default:+ }+}- ctx context.Context- rCtx base.RestoreContext+// extractUpdateColumns extracts column names from SET assignments that belong+// to the original table.+func extractUpdateColumns(setList []*ast.Assignment, database, _ string, matchedTable *TableReference, normalColumns []string) []string {+ var result []string+ for _, assignment := range setList {+ col := assignment.Column+ if col == nil {+ continue+ }- backupDatabase string- backupTable string- originalDatabase string- originalTable string- generatedColumns []string- normalColumns []string- result string- err error-}+ if col.Schema != "" && !strings.EqualFold(col.Schema, database) {+ continue+ }-func (g *generator) EnterDeleteStatement(ctx *parser.DeleteStatementContext) {- if !isTopLevel(ctx.GetParent()) {- return- }+ if col.Table == "" {+ // Unqualified column: check if it belongs to the normalColumns set.+ for _, c := range normalColumns {+ if strings.EqualFold(c, col.Column) {+ result = append(result, col.Column)+ break+ }+ }+ continue+ }- if len(g.generatedColumns) == 0 {- g.result = fmt.Sprintf("INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s`;", g.originalDatabase, g.originalTable, g.backupDatabase, g.backupTable)- } else {- var quotedColumns []string- for _, column := range g.normalColumns {- quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))+ if matchedTable != nil && (col.Table == matchedTable.Alias || strings.EqualFold(col.Table, matchedTable.Table)) {+ result = append(result, col.Column)}- quotedColumnList := strings.Join(quotedColumns, ", ")- g.result = fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s`;", g.originalDatabase, g.originalTable, quotedColumnList, quotedColumnList, g.backupDatabase, g.backupTable)}+ return result}-func (g *generator) hasDisjointUniqueKey(updateColumns []string) (bool, error) {+func hasDisjointUniqueKey(ctx context.Context, rCtx base.RestoreContext, originalDatabase, originalTable string, updateColumns []string) (bool, error) {columnMap := make(map[string]bool)for _, column := range updateColumns {columnMap[strings.ToLower(column)] = true}- if g.rCtx.GetDatabaseMetadataFunc == nil {+ if rCtx.GetDatabaseMetadataFunc == nil {return false, errors.Errorf("GetDatabaseMetadataFunc is nil")}- _, metadata, err := g.rCtx.GetDatabaseMetadataFunc(g.ctx, g.rCtx.InstanceID, g.originalDatabase)+ _, metadata, err := rCtx.GetDatabaseMetadataFunc(ctx, rCtx.InstanceID, originalDatabase)if err != nil {- return false, errors.Wrapf(err, "failed to get database metadata for %s", g.originalDatabase)+ return false, errors.Wrapf(err, "failed to get database metadata for %s", originalDatabase)}-if metadata == nil {- return false, errors.Errorf("database metadata is nil for %s", g.originalDatabase)+ return false, errors.Errorf("database metadata is nil for %s", originalDatabase)}schema := metadata.GetSchemaMetadata("")if schema == nil {- return false, errors.Errorf("schema is nil for %s", g.originalDatabase)+ return false, errors.Errorf("schema is nil for %s", originalDatabase)}- tableMetadata := schema.GetTable(g.originalTable)+ tableMetadata := schema.GetTable(originalTable)if tableMetadata == nil {- return false, errors.Errorf("table metadata is nil for %s.%s", g.originalDatabase, g.originalTable)+ return false, errors.Errorf("table metadata is nil for %s.%s", originalDatabase, originalTable)}for _, index := range tableMetadata.GetProto().Indexes {@@ -249,105 +279,110 @@ func disjoint(a []string, b map[string]bool) bool {return true}-func (g *generator) EnterUpdateStatement(ctx *parser.UpdateStatementContext) {- if !isTopLevel(ctx.GetParent()) {- return+func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {+ if backupItem == nil {+ return "", errors.Errorf("backup item is nil")}- singleTables := &singleTableListener{- databaseName: g.originalDatabase,- singleTables: make(map[string]*TableReference),+ list, err := SplitSQL(statement)+ if err != nil {+ return "", errors.Wrap(err, "failed to split sql")}- antlr.ParseTreeWalkerDefault.Walk(singleTables, ctx.TableReferenceList())-- updateItems := &updateItemListener{- database: g.originalDatabase,- normalColumns: g.normalColumns,+ start := 0+ end := len(list) - 1+ for i, item := range list {+ if equalOrLess(item.Start, backupItem.StartPosition) {+ start = i+ }}- for _, table := range singleTables.singleTables {- if strings.EqualFold(table.Table, g.originalTable) {- updateItems.table = table- break++ for i := len(list) - 1; i >= 0; i-- {+ if equalOrGreater(list[i].Start, backupItem.EndPosition) {+ end = i}}- antlr.ParseTreeWalkerDefault.Walk(updateItems, ctx.UpdateList())- has, err := g.hasDisjointUniqueKey(updateItems.result)+ _, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)if err != nil {- g.err = err- return- }- if !has {- g.err = errors.Errorf("no disjoint unique key found for %s.%s", g.originalDatabase, g.originalTable)- return+ return "", errors.Wrapf(err, "failed to get source database ID for %s", backupItem.SourceTable.Database)}- var buf strings.Builder- if len(g.generatedColumns) == 0 {- if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", g.originalDatabase, g.originalTable, g.backupDatabase, g.backupTable); err != nil {- g.err = err- return+ var result []string+ for i := start; i <= end; i++ {+ stmtList, err := ParseMySQLOmni(list[i].Text)+ if err != nil {+ return "", errors.Wrap(err, "failed to parse sql")}- } else {- var quotedColumns []string- for _, column := range g.normalColumns {- quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))+ containsSourceTable := false+ for _, node := range stmtList.Items {+ if containsTable(node, sourceDatabase, backupItem.SourceTable.Table) {+ containsSourceTable = true+ break+ }}- quotedColumnList := strings.Join(quotedColumns, ", ")- if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", g.originalDatabase, g.originalTable, quotedColumnList, quotedColumnList, g.backupDatabase, g.backupTable); err != nil {- g.err = err- return+ if containsSourceTable {+ result = append(result, list[i].Text)}}+ return strings.Join(result, ""), nil+}- for i, field := range updateItems.result {- if i > 0 {- if _, err := buf.WriteString(", "); err != nil {- g.err = err- return+// containsTable checks whether the given AST node references the specified table.+func containsTable(node ast.Node, database, table string) bool {+ switch n := node.(type) {+ case *ast.UpdateStmt:+ for _, expr := range n.Tables {+ if tableExprReferences(expr, database, table) {+ return true}}-- if _, err := fmt.Fprintf(&buf, "`%s` = VALUES(`%s`)", field, field); err != nil {- g.err = err- return+ case *ast.DeleteStmt:+ for _, expr := range n.Tables {+ if tableExprReferences(expr, database, table) {+ return true+ }}+ for _, expr := range n.Using {+ if tableExprReferences(expr, database, table) {+ return true+ }+ }+ default:}- if _, err := buf.WriteString(";"); err != nil {- g.err = err- return- }- g.result = buf.String()+ return false}-type updateItemListener struct {- *parser.BaseMySQLParserListener- normalColumns []string- database string- table *TableReference- result []string+func tableExprReferences(expr ast.TableExpr, database, table string) bool {+ switch n := expr.(type) {+ case *ast.TableRef:+ db := n.Schema+ if db == "" {+ db = database+ }+ return db == database && strings.EqualFold(n.Name, table)+ case *ast.JoinClause:+ return tableExprReferences(n.Left, database, table) || tableExprReferences(n.Right, database, table)+ }+ return false}-func (l *updateItemListener) EnterUpdateElement(ctx *parser.UpdateElementContext) {- database, table, column := NormalizeMySQLColumnRef(ctx.ColumnRef())-- if database != "" && !strings.EqualFold(database, l.database) {- return+func equalOrLess(a, b *storepb.Position) bool {+ if a.Line < b.Line {+ return true}-- if table == "" {- for _, c := range l.normalColumns {- if strings.EqualFold(c, column) {- l.result = append(l.result, column)- return- }- }- return+ if a.Line == b.Line && a.Column <= b.Column {+ return true}+ return false+}- if l.table.Alias == table || strings.EqualFold(l.table.Table, table) {- l.result = append(l.result, column)- return+func equalOrGreater(a, b *storepb.Position) bool {+ if a.Line > b.Line {+ return true}+ if a.Line == b.Line && a.Column >= b.Column {+ return true+ }+ return false}