Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(parser/tidb): add DML rollback support (closes BYT-7201) (#20345)
Vincent Huang0351d3d866bc
diff --git a/backend/plugin/parser/tidb/backup_test.go b/backend/plugin/parser/tidb/backup_test.goindex 683a42d38b..2c297e1d8f 100644--- a/backend/plugin/parser/tidb/backup_test.go+++ b/backend/plugin/parser/tidb/backup_test.go@@ -99,6 +99,50 @@ func buildFixedMockDatabaseMetadataGetterAndLister() (base.GetDatabaseMetadataFu},},},+ Indexes: []*store.IndexMetadata{+ {+ Name: "PRIMARY",+ Primary: true,+ Unique: true,+ Expressions: []string{+ "b",+ },+ },+ {+ Name: "uk_a",+ Unique: true,+ Expressions: []string{+ "a",+ },+ },+ // Unique key on a generated column (c_generated = a + b).+ // Used by TestGenerateRestoreSQLGeneratedColumnUKSkipped to+ // pin that hasDisjointUniqueKey skips UKs whose+ // expressions reference generated columns. Pre-fix this+ // UK would false-positive as disjoint via naive string+ // comparison; post-fix it's correctly skipped.+ {+ Name: "uk_c_generated",+ Unique: true,+ Expressions: []string{+ "c_generated",+ },+ },+ // Unique key with empty Expressions — represents the+ // TiDB-metadata shape for some expression/functional+ // index parts that don't populate key.Column (per+ // backend/plugin/schema/tidb/get_database_metadata.go).+ // Used by TestGenerateRestoreSQLEmptyExpressionsUKSkipped+ // to pin that hasDisjointUniqueKey skips empty-+ // Expressions UKs. Pre-fix: disjoint([]) returns+ // vacuously true, false-positive disjoint. Post-fix:+ // empty-Expressions UKs are skipped explicitly.+ {+ Name: "uk_empty_expressions",+ Unique: true,+ Expressions: nil,+ },+ },},{Name: "t1",@@ -141,6 +185,22 @@ func buildFixedMockDatabaseMetadataGetterAndLister() (base.GetDatabaseMetadataFuName: "c",},},+ Indexes: []*store.IndexMetadata{+ {+ Name: "PRIMARY",+ Primary: true,+ Expressions: []string{+ "c",+ },+ },+ {+ Name: "PRIMARY",+ Unique: true,+ Expressions: []string{+ "a",+ },+ },+ },},{Name: "test2",diff --git a/backend/plugin/parser/tidb/restore.go b/backend/plugin/parser/tidb/restore.gonew file mode 100644index 0000000000..0de0fc5d0c--- /dev/null+++ b/backend/plugin/parser/tidb/restore.go@@ -0,0 +1,651 @@+package tidb++import (+ "context"+ "fmt"+ "strings"++ "github.com/pkg/errors"++ "github.com/bytebase/omni/tidb/ast"++ "github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++const (+ maxCommentLength = 1000++ // errMsgFailedToGetSourceDB is the wrap-error format string used at+ // every common.GetInstanceDatabaseID(SourceTable.Database) call site.+ // Centralized to avoid string drift across the three error-wrap sites+ // (GenerateRestoreSQL / doGenerate / extractStatement).+ errMsgFailedToGetSourceDB = "failed to get source database ID for %s"+)++func init() {+ base.RegisterGenerateRestoreSQL(storepb.Engine_TIDB, GenerateRestoreSQL)+}++func GenerateRestoreSQL(ctx context.Context, rCtx base.RestoreContext, statement string, backupItem *storepb.PriorBackupDetail_Item) (string, error) {+ // Nil guard: backupItem and its SourceTable/TargetTable sub-fields are+ // dereferenced unconditionally below. Pre-Bug-9 the nil-backupItem check+ // lived in extractStatement (called first); the Bug 9 refactor moved+ // metadata fetching ahead of extractStatement, bypassing that guard.+ // Per Codex P2 catch on PR #20345 — restoring the nil-backupItem check+ // AND adding sub-field checks since each is independently dereferenced.+ if backupItem == nil {+ return "", errors.Errorf("backup item is nil")+ }+ if backupItem.SourceTable == nil {+ return "", errors.Errorf("backup item source table is nil")+ }+ if backupItem.TargetTable == nil {+ return "", errors.Errorf("backup item target table is nil")+ }++ _, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)+ if err != nil {+ return "", errors.Wrapf(err, errMsgFailedToGetSourceDB, backupItem.SourceTable.Database)+ }++ // Fetch the target table's regular column set once. Used by+ // updateMutatesTable to resolve unqualified SET columns against the+ // target's actual schema — without it, an unqualified SET on a+ // non-target-table column would mis-classify the UPDATE as mutating+ // the target and produce invalid `... ON DUPLICATE KEY UPDATE ;`+ // downstream. Per Codex P1 catch on PR #20345.+ targetCols, err := getNormalColumnsLower(ctx, rCtx, sourceDatabase, backupItem.SourceTable.Table)+ if err != nil {+ return "", errors.Wrap(err, "failed to get target table columns")+ }++ originalSQL, err := extractStatement(statement, backupItem, sourceDatabase, targetCols)+ if err != nil {+ return "", errors.Errorf("failed to extract single SQL: %v", err)+ }++ // Find ALL DML nodes referencing the target table — backup.go's single-+ // table path bundles >maxMixedDMLCount same-table DMLs into one backup+ // item, and rollback must cover every column touched across the bundle+ // (not just the first stmt's columns). Per Codex P1 catch on PR #20345.+ matchingNodes, err := findMatchingDMLs(originalSQL, sourceDatabase, backupItem.SourceTable.Table, targetCols)+ if err != nil {+ return "", err+ }+ if len(matchingNodes) == 0 {+ return "", errors.Errorf("no DML statement found in extracted SQL")+ }++ sqlForComment, truncated := common.TruncateString(originalSQL, maxCommentLength)+ if truncated {+ sqlForComment += "..."+ }+ return doGenerate(ctx, rCtx, sqlForComment, matchingNodes, backupItem)+}++// getNormalColumnsLower returns a lowercased set of regular (non-+// generated) column names for the given table. Used by+// updateMutatesTable to determine whether an unqualified SET column+// belongs to the target table — without this, multi-table UPDATEs+// where unqualified SETs reference columns of the joined-but-not-+// target table would be mis-classified as mutating the target.+func getNormalColumnsLower(ctx context.Context, rCtx base.RestoreContext, database, table string) (map[string]bool, error) {+ if rCtx.GetDatabaseMetadataFunc == nil {+ return nil, errors.Errorf("GetDatabaseMetadataFunc is nil")+ }+ _, metadata, err := rCtx.GetDatabaseMetadataFunc(ctx, rCtx.InstanceID, database)+ if err != nil {+ return nil, errors.Wrapf(err, "failed to get database metadata for %s", database)+ }+ if metadata == nil {+ return nil, errors.Errorf("database metadata is nil for %s", database)+ }+ schema := metadata.GetSchemaMetadata("")+ if schema == nil {+ return nil, errors.Errorf("schema is nil for %s", database)+ }+ tableMetadata := schema.GetTable(table)+ if tableMetadata == nil {+ return nil, errors.Errorf("table metadata is nil for %s.%s", database, table)+ }+ result := make(map[string]bool)+ for _, col := range tableMetadata.GetProto().Columns {+ if col == nil || col.Generation != nil {+ continue+ }+ result[strings.ToLower(col.Name)] = true+ }+ return result, nil+}++// findMatchingDMLs returns ALL UPDATE/DELETE nodes in the parsed statement+// list that reference the target table. The single-DML form (return-on-+// first-match) is incorrect for backup.go's single-table-bundling path,+// where one backup item spans multiple same-table DMLs touching different+// columns; rolling back only the first stmt's columns leaves later+// columns mutated. See Codex P1 catch on PR #20345.+func findMatchingDMLs(statement, database, table string, targetCols map[string]bool) ([]ast.Node, error) {+ stmtList, err := ParseTiDBOmni(statement)+ if err != nil {+ return nil, errors.Wrap(err, "failed to parse statement")+ }+ var result []ast.Node+ for _, item := range stmtList.Items {+ switch item.(type) {+ case *ast.UpdateStmt, *ast.DeleteStmt:+ if containsTable(item, database, table, targetCols) {+ result = append(result, item)+ }+ default:+ }+ }+ return result, nil+}++func doGenerate(ctx context.Context, rCtx base.RestoreContext, sqlForComment string, nodes []ast.Node, backupItem *storepb.PriorBackupDetail_Item) (string, error) {+ _, sourceDatabase, err := common.GetInstanceDatabaseID(backupItem.SourceTable.Database)+ if err != nil {+ return "", errors.Wrapf(err, errMsgFailedToGetSourceDB, backupItem.SourceTable.Database)+ }+ _, targetDatabase, err := common.GetInstanceDatabaseID(backupItem.TargetTable.Database)+ if err != nil {+ return "", errors.Wrapf(err, "failed to get target database ID for %s", backupItem.TargetTable.Database)+ }+ generatedColumns, normalColumns, err := classifyColumns(ctx, rCtx.GetDatabaseMetadataFunc, rCtx.ListDatabaseNamesFunc, rCtx.IsCaseSensitive, rCtx.InstanceID, &TableReference{+ Database: sourceDatabase,+ Table: backupItem.SourceTable.Table,+ })+ if err != nil {+ return "", errors.Wrapf(err, "failed to classify columns for %s.%s", backupItem.SourceTable.Database, backupItem.SourceTable.Table)+ }++ // Partition nodes by type. If any UPDATE exists in the bundle, ODKU+ // is required — pure INSERT SELECT would FAIL on the surviving UPDATE-+ // modified rows due to duplicate key. ODKU restores the listed columns+ // for those rows AND inserts the deleted rows (no conflict). If the+ // bundle is pure-DELETE, simple INSERT SELECT suffices (no surviving+ // rows means no conflicts).+ var updateStmts []*ast.UpdateStmt+ for _, n := range nodes {+ switch v := n.(type) {+ case *ast.UpdateStmt:+ updateStmts = append(updateStmts, v)+ case *ast.DeleteStmt:+ // DELETE doesn't contribute columns; tracked only by partition.+ default:+ return "", errors.Errorf("unexpected statement type: %T", n)+ }+ }++ var result string+ if len(updateStmts) > 0 {+ r, err := generateUpdateRestore(ctx, rCtx, updateStmts, sourceDatabase, backupItem.SourceTable.Table, targetDatabase, backupItem.TargetTable.Table, generatedColumns, normalColumns)+ if err != nil {+ return "", err+ }+ result = r+ } else {+ result = generateDeleteRestore(sourceDatabase, backupItem.SourceTable.Table, targetDatabase, backupItem.TargetTable.Table, generatedColumns, normalColumns)+ }++ return fmt.Sprintf("/*\nOriginal SQL:\n%s\n*/\n%s", sqlForComment, result), nil+}++func generateDeleteRestore(originalDatabase, originalTable, backupDatabase, backupTable string, generatedColumns, normalColumns []string) string {+ if len(generatedColumns) == 0 {+ return fmt.Sprintf("INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s`;", originalDatabase, originalTable, backupDatabase, backupTable)+ }+ var quotedColumns []string+ for _, column := range normalColumns {+ quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))+ }+ quotedColumnList := strings.Join(quotedColumns, ", ")+ return fmt.Sprintf("INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s`;", originalDatabase, originalTable, quotedColumnList, quotedColumnList, backupDatabase, backupTable)+}++func generateUpdateRestore(ctx context.Context, rCtx base.RestoreContext, stmts []*ast.UpdateStmt, originalDatabase, originalTable, backupDatabase, backupTable string, generatedColumns, normalColumns []string) (string, error) {+ // Union update columns across ALL stmts in the bundle. backup.go's+ // single-table path bundles >maxMixedDMLCount same-table DMLs into one+ // backup_item; restore must rollback every column touched across the+ // whole bundle, not just the first stmt's columns. Per Codex P1 catch+ // on PR #20345.+ //+ // Iteration order is preserved (first-seen-first) for deterministic+ // ODKU clause output; case-insensitive dedup matches the+ // hasDisjointUniqueKey lower-case map convention.+ seen := make(map[string]bool)+ var updateColumns []string+ for _, stmt := range stmts {+ singleTables := extractSingleTablesFromTableExprs(originalDatabase, stmt.Tables)+ for _, col := range extractUpdateColumns(stmt.SetList, originalDatabase, originalTable, singleTables, normalColumns) {+ key := strings.ToLower(col)+ if !seen[key] {+ seen[key] = true+ updateColumns = append(updateColumns, col)+ }+ }+ }++ has, err := hasDisjointUniqueKey(ctx, rCtx, originalDatabase, originalTable, updateColumns)+ if err != nil {+ return "", err+ }+ if !has {+ return "", errors.Errorf("no disjoint unique key found for %s.%s", originalDatabase, originalTable)+ }++ var buf strings.Builder+ if len(generatedColumns) == 0 {+ if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` SELECT * FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", originalDatabase, originalTable, backupDatabase, backupTable); err != nil {+ return "", err+ }+ } else {+ var quotedColumns []string+ for _, column := range normalColumns {+ quotedColumns = append(quotedColumns, fmt.Sprintf("`%s`", column))+ }+ quotedColumnList := strings.Join(quotedColumns, ", ")+ if _, err := fmt.Fprintf(&buf, "INSERT INTO `%s`.`%s` (%s) SELECT %s FROM `%s`.`%s` ON DUPLICATE KEY UPDATE ", originalDatabase, originalTable, quotedColumnList, quotedColumnList, backupDatabase, backupTable); err != nil {+ return "", err+ }+ }++ for i, field := range updateColumns {+ if i > 0 {+ if _, err := buf.WriteString(", "); err != nil {+ return "", err+ }+ }+ if _, err := fmt.Fprintf(&buf, "`%s` = VALUES(`%s`)", field, field); err != nil {+ return "", err+ }+ }+ if _, err := buf.WriteString(";"); err != nil {+ return "", err+ }+ return buf.String(), nil+}++// extractSingleTablesFromTableExprs walks omni TableExpr nodes and returns+// a map of alias-or-name to TableReference for simple table references.+func extractSingleTablesFromTableExprs(databaseName string, exprs []ast.TableExpr) map[string]*TableReference {+ result := make(map[string]*TableReference)+ for _, expr := range exprs {+ collectTableRefs(databaseName, expr, result)+ }+ return result+}++func collectTableRefs(databaseName string, expr ast.TableExpr, result map[string]*TableReference) {+ switch n := expr.(type) {+ case *ast.TableRef:+ database := n.Schema+ if database == "" {+ database = databaseName+ }+ table := &TableReference{+ Database: database,+ Table: n.Name,+ Alias: n.Alias,+ }+ // Map keys are normalized to lowercase to match TiDB's+ // case-insensitive identifier semantics — a SET clause that+ // references the alias with different case (e.g., `T1` declared,+ // `t1` referenced in SET) must still resolve. Per Codex P1+ // follow-on catch on PR #20345.+ var key string+ if n.Alias != "" {+ key = strings.ToLower(n.Alias)+ } else {+ key = strings.ToLower(n.Name)+ }+ result[key] = table+ case *ast.JoinClause:+ collectTableRefs(databaseName, n.Left, result)+ collectTableRefs(databaseName, n.Right, result)+ default:+ }+}++// extractUpdateColumns extracts column names from SET assignments that+// belong to the original table.+//+// For qualified SET columns (`SET t1.a = ...`), the qualifier is looked+// up in singleTables (the alias-or-name → TableReference map for this+// stmt) — if the qualifier resolves to a TableReference whose .Table+// equals originalTable, the column is included. This is deterministic+// even for self-join UPDATEs where the same physical table appears+// under multiple aliases (e.g., `UPDATE test t1 JOIN test t2 SET t1.a+// = ...`); each SET clause's qualifier independently resolves to the+// correct alias entry. Per Codex P1 catch on PR #20345.+//+// Pre-fix this function took a single matchedTable picked by the caller+// via map iteration over singleTables. Map iteration is randomized in+// Go, so for self-joins the wrong alias could be picked, leading to+// empty result and invalid `... ON DUPLICATE KEY UPDATE ;` rollback SQL.+func extractUpdateColumns(setList []*ast.Assignment, database, originalTable string, singleTables map[string]*TableReference, normalColumns []string) []string {+ var result []string+ for _, assignment := range setList {+ col := assignment.Column+ if col == nil {+ continue+ }++ if col.Schema != "" && !strings.EqualFold(col.Schema, database) {+ continue+ }++ if col.Table == "" {+ // Unqualified column: check if it belongs to the normalColumns set.+ for _, c := range normalColumns {+ if strings.EqualFold(c, col.Column) {+ result = append(result, col.Column)+ break+ }+ }+ continue+ }++ // Qualified column: resolve the qualifier through the alias map+ // (handles self-joins deterministically). Lookup key is+ // lowercased to match the case-insensitive insert convention+ // in collectTableRefs. BOTH Database AND Table must match —+ // without the Database check, cross-database joins with+ // homonymous tables (e.g. `UPDATE db1.test t1 JOIN db2.test t2+ // SET t2.a = ...`) would incorrectly include the joined-DB+ // SET column in the target-DB's rollback. Per Codex P1 catch+ // on PR #20345.+ if entry, ok := singleTables[strings.ToLower(col.Table)]; ok && strings.EqualFold(entry.Database, database) && strings.EqualFold(entry.Table, originalTable) {+ result = append(result, col.Column)+ continue+ }+ // Fallback: qualifier IS the bare table name (no alias used).+ // col.Schema check above already filtered explicit-schema+ // mismatches.+ if strings.EqualFold(col.Table, originalTable) {+ result = append(result, col.Column)+ }+ }+ return result+}++func hasDisjointUniqueKey(ctx context.Context, rCtx base.RestoreContext, originalDatabase, originalTable string, updateColumns []string) (bool, error) {+ columnMap := make(map[string]bool)+ for _, column := range updateColumns {+ columnMap[strings.ToLower(column)] = true+ }++ if rCtx.GetDatabaseMetadataFunc == nil {+ return false, errors.Errorf("GetDatabaseMetadataFunc is nil")+ }++ _, metadata, err := rCtx.GetDatabaseMetadataFunc(ctx, rCtx.InstanceID, originalDatabase)+ if err != nil {+ return false, errors.Wrapf(err, "failed to get database metadata for %s", originalDatabase)+ }+ if metadata == nil {+ return false, errors.Errorf("database metadata is nil for %s", originalDatabase)+ }++ schema := metadata.GetSchemaMetadata("")+ if schema == nil {+ return false, errors.Errorf("schema is nil for %s", originalDatabase)+ }++ tableMetadata := schema.GetTable(originalTable)+ if tableMetadata == nil {+ return false, errors.Errorf("table metadata is nil for %s.%s", originalDatabase, originalTable)+ }++ tableProto := tableMetadata.GetProto()++ // Build a fast lookup of regular (non-generated) column names. Used+ // to filter out unique keys whose Expressions reference generated+ // columns or non-column expressions (functional indexes) — those are+ // unsafe for ODKU rollback because string-comparison disjoint can't+ // tell whether the SET clause indirectly affects them. Per peer+ // review on PR #20345 (Finding 4).+ regularColumns := make(map[string]bool)+ for _, col := range tableProto.Columns {+ if col == nil || col.Generation != nil {+ continue+ }+ regularColumns[strings.ToLower(col.Name)] = true+ }++ for _, index := range tableProto.Indexes {+ if !index.Primary && !index.Unique {+ continue+ }+ // Skip UKs whose Expressions reference anything other than+ // regular (non-generated) columns. A UK on a generated column+ // (e.g., `c_generated = a + b`) appears disjoint from SET cols+ // {a, b} via string comparison — but updating a or b changes+ // c_generated's value, so the UK is NOT safe for ODKU matching.+ // Same problem for functional indexes (Expressions = `(LOWER(email))`).+ // Conservative: treat any UK with non-regular-column expressions+ // as overlapping (skip). Never returns false-positive disjoint.+ //+ // ALSO skip UKs with empty Expressions — disjoint([]) returns+ // vacuously true, which would falsely mark the UK as safe.+ // TiDB metadata produces empty Expressions for some expression-+ // based index parts (per backend/plugin/schema/tidb/+ // get_database_metadata.go:getIndexColumnsInfo, parts without+ // key.Column aren't added). Per Codex P1 catch on PR #20345.+ if len(index.Expressions) == 0 {+ continue+ }+ safe := true+ for _, expr := range index.Expressions {+ if !regularColumns[strings.ToLower(expr)] {+ safe = false+ break+ }+ }+ if !safe {+ continue+ }+ if disjoint(index.Expressions, columnMap) {+ return true, nil+ }+ }++ return false, nil+}++func disjoint(a []string, b map[string]bool) bool {+ for _, item := range a {+ if _, ok := b[strings.ToLower(item)]; ok {+ return false+ }+ }+ return true+}++func extractStatement(statement string, backupItem *storepb.PriorBackupDetail_Item, sourceDatabase string, targetCols map[string]bool) (string, error) {+ // Nil-backupItem guard now lives in GenerateRestoreSQL (the only+ // caller); extractStatement is unexported and assumes non-nil input.++ list, err := SplitSQL(statement)+ if err != nil {+ return "", errors.Wrap(err, "failed to split sql")+ }++ start := 0+ end := len(list) - 1+ for i, item := range list {+ if equalOrLess(item.Start, backupItem.StartPosition) {+ start = i+ }+ }++ for i := len(list) - 1; i >= 0; i-- {+ // EndPosition is the EXCLUSIVE end of the range (per+ // base/statement.go:16-18, "points to the position AFTER the last+ // character of the statement"). A stmt whose Start is AT or AFTER+ // EndPosition belongs to the NEXT backup item, not this one — so+ // it must be EXCLUDED from this slice. Pre-fix used `end = i`+ // which included the boundary stmt; in mixed-DML mode (each+ // backup item maps to one stmt), that bled the next backup+ // item's stmt into this one's rollback. Per Codex P1 catch on+ // PR #20345.+ if equalOrGreater(list[i].Start, backupItem.EndPosition) {+ end = i - 1+ }+ }++ var result []string+ for i := start; i <= end; i++ {+ stmtList, err := ParseTiDBOmni(list[i].Text)+ if err != nil {+ return "", errors.Wrap(err, "failed to parse sql")+ }+ containsSourceTable := false+ for _, node := range stmtList.Items {+ if containsTable(node, sourceDatabase, backupItem.SourceTable.Table, targetCols) {+ containsSourceTable = true+ break+ }+ }+ if containsSourceTable {+ result = append(result, list[i].Text)+ }+ }+ return strings.Join(result, ""), nil+}++// containsTable checks whether the given AST node MUTATES the specified+// table (not merely references it via JOIN/USING).+//+// For UPDATE: the table must appear as the qualifier of at least one SET+// assignment — being only joined for filtering is not enough.+//+// For DELETE: n.Tables holds the explicit delete-targets (mutation set);+// n.Using holds JOIN-only refs (filter set). Only n.Tables counts as+// mutation. Per Codex P1 follow-on catch on PR #20345 — pre-fix+// matching n.Using too caused a backup item targeting a USING-only+// table to generate rollback SQL that re-inserted rows that were never+// deleted (reintroducing stale data).+func containsTable(node ast.Node, database, table string, targetCols map[string]bool) bool {+ switch n := node.(type) {+ case *ast.UpdateStmt:+ return updateMutatesTable(n, database, table, targetCols)+ case *ast.DeleteStmt:+ for _, expr := range n.Tables {+ if tableExprReferences(expr, database, table) {+ return true+ }+ }+ // n.Using is JOIN-only (filter set), not mutation — do NOT match.+ default:+ }+ return false+}++// updateMutatesTable reports whether the UPDATE's SET clauses actually+// mutate (a column of) the specified table. JOIN-only references that+// do not appear as a SET-clause qualifier do NOT count as mutation.+//+// Resolution strategy mirrors extractUpdateColumns: build the alias map+// (lowercased per Bug 5), then for each SET assignment resolve the+// qualifier through the map. Unqualified columns are conservatively+// counted as mutation if the target table is in scope (multi-table+// UPDATE with unqualified col is ambiguous; safer to over-include than+// miss).+func updateMutatesTable(stmt *ast.UpdateStmt, database, table string, targetCols map[string]bool) bool {+ // Precondition: target must be in stmt.Tables at all (else there's+ // nothing to discuss).+ targetInScope := false+ for _, expr := range stmt.Tables {+ if tableExprReferences(expr, database, table) {+ targetInScope = true+ break+ }+ }+ if !targetInScope {+ return false+ }++ singleTables := extractSingleTablesFromTableExprs(database, stmt.Tables)+ for _, assignment := range stmt.SetList {+ col := assignment.Column+ if col == nil {+ continue+ }+ if col.Schema != "" && !strings.EqualFold(col.Schema, database) {+ continue+ }+ if col.Table == "" {+ // Unqualified column. Resolve against the target table's+ // actual columns — only count as mutation if the column+ // exists on the target. Pre-fix this branch returned true+ // unconditionally (over-counted); for `UPDATE test JOIN t1+ // ON ... SET name = 1` where `name` is on t1 but not test,+ // the over-count produced empty `... ON DUPLICATE KEY+ // UPDATE ;`. Per Codex P1 catch on PR #20345.+ if targetCols[strings.ToLower(col.Column)] {+ return true+ }+ continue+ }+ // Qualified — resolve qualifier through the alias map. Both+ // Database AND Table must match: for cross-database joins with+ // homonymous tables (e.g. `UPDATE db1.test t1 JOIN db2.test t2+ // SET t2.a = ...`), the alias t2 resolves to db2.test; without+ // the Database check, a backup item targeting db1.test would+ // incorrectly match db2.test's SET assignments. Per Codex P1+ // catch on PR #20345.+ if entry, ok := singleTables[strings.ToLower(col.Table)]; ok && strings.EqualFold(entry.Database, database) && strings.EqualFold(entry.Table, table) {+ return true+ }+ // Fallback: qualifier IS the bare table name (no alias used).+ // The col.Schema check at the top of the loop already filtered+ // out assignments whose explicit schema doesn't match `database`,+ // so reaching here implies col.Schema is empty or matches.+ if strings.EqualFold(col.Table, table) {+ return true+ }+ }+ return false+}++func tableExprReferences(expr ast.TableExpr, database, table string) bool {+ switch n := expr.(type) {+ case *ast.TableRef:+ db := n.Schema+ if db == "" {+ db = database+ }+ // Both comparisons are case-insensitive — TiDB/MySQL identifier+ // comparisons are typically case-insensitive in practice, and the+ // table-name comparison below already uses EqualFold; using == on+ // the database side would inconsistently miss schema-qualified+ // references like `DB.test` when backupItem stores `db`.+ return strings.EqualFold(db, database) && strings.EqualFold(n.Name, table)+ case *ast.JoinClause:+ return tableExprReferences(n.Left, database, table) || tableExprReferences(n.Right, database, table)+ }+ return false+}++func equalOrLess(a, b *storepb.Position) bool {+ if a.Line < b.Line {+ return true+ }+ if a.Line == b.Line && a.Column <= b.Column {+ return true+ }+ return false+}++func equalOrGreater(a, b *storepb.Position) bool {+ if a.Line > b.Line {+ return true+ }+ if a.Line == b.Line && a.Column >= b.Column {+ return true+ }+ return false+}diff --git a/backend/plugin/parser/tidb/restore_test.go b/backend/plugin/parser/tidb/restore_test.gonew file mode 100644index 0000000000..d8e2782927--- /dev/null+++ b/backend/plugin/parser/tidb/restore_test.go@@ -0,0 +1,720 @@+package tidb++import (+ "context"+ "io"+ "math"+ "os"+ "testing"++ "github.com/stretchr/testify/require"+ "gopkg.in/yaml.v3"++ "github.com/bytebase/bytebase/backend/common/yamltest"+ "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++type restoreCase struct {+ Input string+ BackupDatabase string+ BackupTable string+ OriginalDatabase string+ OriginalTable string+ Result string+}++func TestRestore(t *testing.T) {+ tests := []restoreCase{}++ const (+ record = false+ )+ var (+ filepath = "test-data/test_restore.yaml"+ )++ a := require.New(t)+ yamlFile, err := os.Open(filepath)+ a.NoError(err)++ byteValue, err := io.ReadAll(yamlFile)+ a.NoError(yamlFile.Close())+ a.NoError(err)+ a.NoError(yaml.Unmarshal(byteValue, &tests))++ for i, t := range tests {+ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, t.Input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/" + t.OriginalDatabase,+ Table: t.OriginalTable,+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/" + t.BackupDatabase,+ Table: t.BackupTable,+ },+ StartPosition: &store.Position{+ Line: 0,+ Column: 0,+ },+ EndPosition: &store.Position{+ Line: math.MaxInt32,+ Column: 0,+ },+ })+ a.NoError(err)++ if record {+ tests[i].Result = result+ } else {+ a.Equal(t.Result, result, t.Input)+ }+ }+ if record {+ yamltest.Record(t, filepath, tests)+ }+}++func TestTiDBGenerateRestoreSQLRegistration(t *testing.T) {+ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()++ result, err := base.GenerateRestoreSQL(context.Background(), store.Engine_TIDB, base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, "DELETE FROM test WHERE b1 = 1;", &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_test",+ },+ StartPosition: &store.Position{+ Line: 0,+ Column: 0,+ },+ EndPosition: &store.Position{+ Line: math.MaxInt32,+ Column: 0,+ },+ })++ require.NoError(t, err)+ require.Equal(t, "/*\nOriginal SQL:\nDELETE FROM test WHERE b1 = 1;\n*/\nINSERT INTO `db`.`test` SELECT * FROM `bbarchive`.`prefix_test`;", result)+}++// TestGenerateRestoreSQLCaseInsensitiveDatabaseQualifier pins the+// case-insensitive database-qualifier match contract in+// tableExprReferences. TiDB/MySQL identifier comparisons are typically+// case-insensitive in practice; the table-name side already uses+// EqualFold. Codex P2 catch on PR #20345 — the database side previously+// used == which silently missed schema-qualified references where the+// SQL's qualifier case differed from the backup item's stored case.+//+// Construct: SQL uses uppercase `DB.test`; backupItem stores lowercase+// `db`. Pre-fix, containsTable returned false, extractStatement returned+// empty, and the customer saw "no DML statement found" instead of the+// expected rollback SQL.+func TestGenerateRestoreSQLCaseInsensitiveDatabaseQualifier(t *testing.T) {+ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()++ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, "DELETE FROM DB.test WHERE c = 1;", &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.NoError(t, err,+ "case-mismatched database qualifier (DB vs db) must still match — TiDB identifier comparison is case-insensitive in practice")+ require.Contains(t, result, "INSERT INTO `db`.`test`",+ "generated rollback should target the lowercase database name from the backup item")+}++// TestGenerateRestoreSQLSelfJoinUpdate pins deterministic alias resolution+// for self-join UPDATEs. Per Codex P1 catch on PR #20345.+//+// Pre-fix bug: extractSingleTablesFromTableExprs returns+// map[alias]*TableReference; the loop in generateUpdateRestore picked+// matchedTable by ranging over that map (Go map iteration is randomized).+// For a self-join `UPDATE test t1 JOIN test t2 ...`, both t1 and t2+// satisfy the `table.Table == originalTable` check, so either could be+// picked. If t2 was picked but the SET clause only references t1, then+// extractUpdateColumns returned EMPTY (col.Table="t1" doesn't match+// matchedTable.Alias="t2", and col.Table doesn't equal-fold matchedTable.+// Table="test" either). Empty updateColumns produces invalid rollback+// SQL: `... ON DUPLICATE KEY UPDATE ;` (semicolon directly after UPDATE).+//+// Fix: extractUpdateColumns now takes the full singleTables map; for+// each qualified SET column it looks up the qualifier directly to+// determine whether the qualifier resolves to the original table —+// no need to pre-pick a single matchedTable.+//+// We loop the test 50 times to expose the nondeterminism if the fix+// regresses; one iteration is sufficient post-fix (deterministic).+func TestGenerateRestoreSQLSelfJoinUpdate(t *testing.T) {+ const input = "UPDATE test t1 JOIN test t2 ON t1.c = t2.c SET t1.a = 1 WHERE t1.c = 1;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()++ for i := 0; i < 50; i++ {+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.NoError(t, err, "iteration %d: GenerateRestoreSQL must succeed for self-join UPDATE", i)+ require.Contains(t, result, "`a` = VALUES(`a`)",+ "iteration %d: ODKU must mention `a` (from SET t1.a = 1); empty ODKU produces invalid SQL", i)+ require.NotContains(t, result, "ON DUPLICATE KEY UPDATE ;",+ "iteration %d: ODKU clause must not be empty (would be invalid TiDB SQL)", i)+ }+}++// TestGenerateRestoreSQLSelfJoinMixedCaseAlias pins case-insensitive+// alias-map matching. Per Codex P1 follow-on catch on PR #20345 (review+// of the Fix #4 self-join determinism patch).+//+// Pre-fix bug: collectTableRefs stored map keys verbatim from+// omni AST (preserving the user's case), and extractUpdateColumns did+// an exact-string map lookup. TiDB ACCEPTS statements like+// `UPDATE test T1 JOIN test T2 ON T1.c = T2.c SET t1.a = 1`+// (alias referenced with different case in SET), but the map lookup+// `singleTables["t1"]` against entries keyed "T1"/"T2" missed.+// Fallback `EqualFold(col.Table, originalTable)` also missed+// (`EqualFold("t1","test") == false`). Result: empty updateColumns+// → invalid `... ON DUPLICATE KEY UPDATE ;`.+//+// Codex validated TiDB acceptance by executing the input against+// pingcap/tidb:v8.5.5 directly; mixed-case aliases are valid TiDB+// SQL.+//+// Fix: normalize map keys to lowercase on insert AND lookup.+func TestGenerateRestoreSQLSelfJoinMixedCaseAlias(t *testing.T) {+ const input = "UPDATE test T1 JOIN test T2 ON T1.c = T2.c SET t1.a = 1 WHERE T1.c = 1;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.NoError(t, err)+ require.Contains(t, result, "`a` = VALUES(`a`)",+ "ODKU must mention `a` (from SET t1.a = 1) — alias-map lookup must be case-insensitive")+ require.NotContains(t, result, "ON DUPLICATE KEY UPDATE ;",+ "ODKU clause must not be empty (would be invalid TiDB SQL)")+}++// TestGenerateRestoreSQLJoinedNotMutated pins that an UPDATE which+// references the target table only via JOIN (not as a SET-clause+// qualifier) does NOT generate rollback SQL for that table. Per peer+// review on PR #20345.+//+// Pre-fix bug: containsTable matched the target if it appeared in+// n.Tables (which includes JOIN-only refs). For+// `UPDATE test2 JOIN test ON ... SET test2.a = ...` with backupItem+// targeting `test`, the match incorrectly fired; extractUpdateColumns+// then correctly returned no `test`-table columns; doGenerate emitted+// invalid `... ON DUPLICATE KEY UPDATE ;` for `test`.+//+// Post-fix: containsTable's UpdateStmt arm uses updateMutatesTable,+// which checks SET-clause qualifiers (resolved through the alias map)+// — JOIN-only refs no longer trigger rollback for the wrong table.+func TestGenerateRestoreSQLJoinedNotMutated(t *testing.T) {+ const input = "UPDATE test2 JOIN test ON test2.c = test.c SET test2.a = 1 WHERE test2.c = 1;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test", // backup targets test, but test is JOINED not UPDATED+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ // The UPDATE doesn't mutate `test` (only joins it), so no DML matches+ // the backup_item's target. Pre-fix, this produced invalid SQL with+ // empty ODKU; post-fix, the "no DML statement found" error is the+ // correct outcome (there's genuinely no DML to roll back for `test`).+ require.Error(t, err,+ "backup item for joined-only-not-mutated table must not generate (invalid) rollback SQL")+ require.Contains(t, err.Error(), "no DML statement found",+ "expected no-DML error since test is only joined, not mutated")+}++// TestGenerateRestoreSQLGeneratedColumnUKSkipped pins that+// hasDisjointUniqueKey skips unique keys whose Expressions reference+// generated columns (or non-column expressions like functional indexes).+// Per peer review on PR #20345 (Finding 4).+//+// Pre-fix bug: disjoint() did naive string comparison between+// index.Expressions and the SET-columns map. For a UK on c_generated+// (where c_generated = a + b), updating columns {a, b} appeared+// "disjoint" from the UK's Expressions ["c_generated"] — string `c_generated`+// is not in {a, b}. But updating a or b CHANGES c_generated's value,+// so the UK is NOT safe for ON DUPLICATE KEY UPDATE matching: pre-fix+// would generate rollback SQL that silently fails to match rows.+//+// Post-fix: hasDisjointUniqueKey filters out UKs whose Expressions+// don't all map to regular (non-generated) columns. For this input,+// PK on b and UK on a both overlap with SET {a, b}; the UK on+// c_generated is skipped (generated). No disjoint UK remains, so+// the function returns the "no disjoint unique key found" error+// instead of generating unsafe rollback SQL.+//+// Mock setup: t_generated has PK on b, UK on a, AND a new UK on+// c_generated (added in backup_test.go for this test).+func TestGenerateRestoreSQLGeneratedColumnUKSkipped(t *testing.T) {+ const input = "UPDATE t_generated SET a = 1, b = 2 WHERE c_generated = 3;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "t_generated",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_t_generated",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.Error(t, err,+ "UPDATE that overlaps every regular UK must surface a no-disjoint-key error — the c_generated UK is not safe (its value depends on the SET'd columns)")+ require.Contains(t, err.Error(), "no disjoint unique key found",+ "error must come from hasDisjointUniqueKey, not a different failure mode")+}++// TestGenerateRestoreSQLDeleteUsingOnlyTable pins that a DELETE which+// references the target table only via USING (not as a delete-target in+// n.Tables) does NOT generate rollback SQL for that table. Per Codex+// P1 follow-on catch on PR #20345 — symmetric to the UPDATE-side+// joined-not-mutated fix in commit c4042da055.+//+// Pre-fix bug: containsTable's DeleteStmt arm matched n.Tables OR+// n.Using. For `DELETE test FROM test, test2 as t2 ...` with+// backupItem targeting `test2`, the match incorrectly fired (test2 is+// only in the USING-equivalent filter set). doGenerate emitted+// `INSERT INTO test2 SELECT * FROM bbarchive.prefix_test2` — which+// would re-introduce stale data if applied as rollback (test2 was+// never actually deleted).+//+// Post-fix: n.Tables is the explicit delete-target set; n.Using is+// filter-only and no longer triggers a match.+func TestGenerateRestoreSQLDeleteUsingOnlyTable(t *testing.T) {+ const input = "DELETE test FROM test, test2 as t2 where test.id = t2.id;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test2", // backup targets USING-only table+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test2",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.Error(t, err,+ "backup item for USING-only-not-deleted table must not generate (re-stale) rollback SQL")+ require.Contains(t, err.Error(), "no DML statement found",+ "expected no-DML error since test2 is only joined for filter, not deleted")+}++// TestGenerateRestoreSQLUnqualifiedSetNonTargetColumn pins that an+// unqualified SET column that does NOT belong to the target table does+// NOT trigger rollback for the target. Per Codex P1 catch on PR #20345+// — follow-on to the Bug 6 fix.+//+// Pre-fix bug: updateMutatesTable's unqualified branch returned true+// for any unqualified SET when the target was in scope, regardless of+// whether the column actually existed on the target. For+// `UPDATE test JOIN t1 ... SET name = 1` (where `name` exists on the+// joined table but not on `test`), the over-classification made+// containsTable return true; extractUpdateColumns correctly returned+// no `test`-table columns; doGenerate emitted invalid+// `... ON DUPLICATE KEY UPDATE ;` for `test`.+//+// Post-fix: updateMutatesTable resolves unqualified SET columns+// against the target table's actual normal column set (fetched once+// at the top of GenerateRestoreSQL via getNormalColumnsLower). Non-+// target columns no longer trigger a match.+func TestGenerateRestoreSQLUnqualifiedSetNonTargetColumn(t *testing.T) {+ // `name` doesn't exist on `test` (mock has only a, b, c).+ const input = "UPDATE test JOIN t1 ON test.c = t1.c SET name = 1 WHERE test.c = 1;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.Error(t, err,+ "unqualified SET on non-target-table column must not generate (invalid) rollback SQL")+ require.Contains(t, err.Error(), "no DML statement found",+ "expected no-DML error since SET column doesn't exist on target")+}++// TestGenerateRestoreSQLUnqualifiedSetTargetColumn pins the positive-+// path counterpart: when the unqualified SET column DOES exist on the+// target table, rollback is correctly generated (column resolves+// through targetCols and `extractUpdateColumns` finds it in+// normalColumns).+func TestGenerateRestoreSQLUnqualifiedSetTargetColumn(t *testing.T) {+ // Single-table UPDATE with unqualified SET — the common case.+ // `a` exists on `test` (mock has a, b, c). targetCols includes+ // "a" → updateMutatesTable returns true; extractUpdateColumns+ // returns ["a"]; ODKU UPDATE `a` = VALUES(`a`).+ const input = "UPDATE test SET a = 1 WHERE c = 1;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.NoError(t, err)+ require.Contains(t, result, "`a` = VALUES(`a`)",+ "unqualified SET on target's column must produce ODKU clause for that column")+}++// TestGenerateRestoreSQLCrossDatabaseAliasResolution pins that the+// alias-map qualifier resolution checks BOTH Database AND Table when+// matching a SET clause against the backup target. Per Codex P1 catch+// on PR #20345.+//+// Pre-fix bug: the alias-map lookup compared only entry.Table. For a+// cross-database join with homonymous tables —+//+// UPDATE db.test t1 JOIN otherdb.test t2 ON t1.c = t2.c SET t2.a = 1 WHERE t1.c = 1;+//+// — alias `t2` resolves to {Database: "otherdb", Table: "test"}.+// updateMutatesTable's condition `entry.Table == "test"` matched+// (because the table NAME is "test"), so a backup item targeting+// db.test mis-classified the UPDATE as mutating db.test even though+// the SET only touched otherdb.test. extractUpdateColumns had the+// same bug, so the rollback would have been generated for the wrong+// table.+//+// Post-fix: both branches require `entry.Database == database` AND+// `entry.Table == table` (case-insensitive). Cross-DB SETs no longer+// match.+func TestGenerateRestoreSQLCrossDatabaseAliasResolution(t *testing.T) {+ const input = "UPDATE db.test t1 JOIN otherdb.test t2 ON t1.c = t2.c SET t2.a = 1 WHERE t1.c = 1;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db", // backup targets db.test+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.Error(t, err,+ "cross-DB alias (t2 → otherdb.test) must not match a backup item targeting db.test")+ require.Contains(t, err.Error(), "no DML statement found",+ "expected no-DML error since SET is on otherdb.test, not db.test")+}++// TestGenerateRestoreSQLNilBackupItemGuards pins that GenerateRestoreSQL+// returns a clean error (not a panic) when backupItem or its sub-fields+// are nil. Per Codex P2 catch on PR #20345 — Bug 9's plumbing refactor+// moved metadata-fetching ahead of extractStatement (which previously+// held the nil guard), causing the function to panic on nil input.+//+// Three independent nil paths covered: backupItem itself, SourceTable,+// TargetTable.+func TestGenerateRestoreSQLNilBackupItemGuards(t *testing.T) {+ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ rCtx := base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }++ cases := []struct {+ name string+ backupItem *store.PriorBackupDetail_Item+ wantErr string+ }{+ {+ name: "nil backupItem",+ backupItem: nil,+ wantErr: "backup item is nil",+ },+ {+ name: "nil SourceTable",+ backupItem: &store.PriorBackupDetail_Item{+ SourceTable: nil,+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ },+ wantErr: "backup item source table is nil",+ },+ {+ name: "nil TargetTable",+ backupItem: &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: nil,+ },+ wantErr: "backup item target table is nil",+ },+ }++ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ require.NotPanics(t, func() {+ _, err := GenerateRestoreSQL(context.Background(), rCtx, "DELETE FROM test;", tc.backupItem)+ require.Error(t, err)+ require.Contains(t, err.Error(), tc.wantErr)+ }, "must return error, not panic, on nil input")+ })+ }+}++// TestGenerateRestoreSQLEmptyExpressionsUKSkipped pins that+// hasDisjointUniqueKey skips unique keys with empty Expressions. Per+// Codex P1 catch on PR #20345.+//+// Pre-fix bug: `disjoint([], anyMap)` returns vacuously true (the+// for-loop iterates zero times). So a UK with empty Expressions+// passed through the disjoint check as "safe" — even though it has+// no actual columns to match against. TiDB metadata produces empty+// Expressions for some expression-based index parts (per+// backend/plugin/schema/tidb/get_database_metadata.go's+// getIndexColumnsInfo, parts without key.Column aren't appended).+// Pre-fix the empty-Expressions UK would short-circuit+// hasDisjointUniqueKey to true → unsafe rollback SQL generated.+//+// Post-fix: explicit empty-Expressions check at the top of each UK+// iteration; such UKs are skipped (treated as overlapping).+//+// Mock setup (backup_test.go): t_generated has PK on b, UK on a,+// UK on c_generated (skipped per Bug 7), AND a new UK with empty+// Expressions. The test UPDATE overlaps the regular UKs (a and b+// in SET); pre-fix the empty-Expressions UK would false-positive+// as disjoint; post-fix all UKs are correctly classified as+// overlapping/unsafe → "no disjoint unique key found" error.+func TestGenerateRestoreSQLEmptyExpressionsUKSkipped(t *testing.T) {+ const input = "UPDATE t_generated SET a = 1, b = 2 WHERE c_generated = 3;"++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "t_generated",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_t_generated",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.Error(t, err,+ "empty-Expressions UK must NOT be treated as disjoint (vacuous truth bug)")+ require.Contains(t, err.Error(), "no disjoint unique key found",+ "all UKs must be classified as overlapping/unsafe — empty-Expressions UK skipped explicitly")+}++// TestGenerateRestoreSQLEndPositionExclusive pins the boundary semantic+// of backupItem.EndPosition: it is the EXCLUSIVE end (per+// base/statement.go:16-18, "points to the position AFTER the last+// character of the statement"). extractStatement's slice MUST exclude+// stmts whose Start position equals EndPosition — those belong to the+// NEXT backup item. Per Codex P1 catch on PR #20345.+//+// Why this matters now: pre-Fix-1 (multi-DML union), findFirstDML+// returned only the first DML, so even when extractStatement bled into+// the next stmt, only the first stmt's columns reached the rollback SQL.+// Post-Fix-1, findMatchingDMLs returns ALL matching DMLs from the+// extraction — so the boundary bleed now contributes columns from the+// NEXT backup item to the union, producing wrong rollback SQL (extra+// ODKU columns OR false "no disjoint unique key" errors).+//+// Construct: 2 same-table UPDATEs in one input. Use SplitSQL to get the+// actual position where stmt[0] ends; set backupItem.EndPosition to+// exactly that (mixed-DML mode where each backup item maps to ONE+// stmt). Assert the rollback ODKU mentions ONLY stmt[0]'s column (`a`)+// — not stmt[1]'s column (`b`).+func TestGenerateRestoreSQLEndPositionExclusive(t *testing.T) {+ const input = "UPDATE test SET a = 1 WHERE c = 1;\nUPDATE test SET b = 2 WHERE c = 2;"++ // Get actual stmt boundaries from the splitter.+ stmts, err := SplitSQL(input)+ require.NoError(t, err)+ require.GreaterOrEqual(t, len(stmts), 2, "splitter must produce at least 2 stmts")+ require.NotNil(t, stmts[0].End, "stmt[0].End must be set by splitter")++ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()+ result, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, input, &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ // Mixed-DML mode: this backup item covers ONLY stmt[0]. Setting+ // EndPosition to stmt[0].End (exclusive) MUST exclude stmt[1].+ EndPosition: stmts[0].End,+ })++ require.NoError(t, err)+ require.Contains(t, result, "`a` = VALUES(`a`)",+ "ODKU must contain stmt[0]'s column `a`")+ require.NotContains(t, result, "`b` = VALUES(`b`)",+ "ODKU must NOT contain stmt[1]'s column `b` — that stmt belongs to the NEXT backup item; including it would generate rollback for the wrong scope")+}++// TestGenerateRestoreSQLNoDisjointUniqueKey pins the negative path that the+// yaml golden tests cannot cover (their format is success-only — no+// WantError field). Rolling back an UPDATE requires at least one unique+// key whose columns are NOT in the SET clause; otherwise ON DUPLICATE KEY+// UPDATE has nothing to match against and the rollback is unsafe.+//+// Construct: the `test` table has PK on `c` and a unique key on `a`. An+// UPDATE that touches BOTH `a` and `c` overlaps every unique key, so+// hasDisjointUniqueKey returns false and generateUpdateRestore must+// surface the "no disjoint unique key found" error.+//+// The mysql analog (mysql/restore_test.go) currently lacks symmetric+// coverage; worth a small follow-up to mirror this test there for parity.+func TestGenerateRestoreSQLNoDisjointUniqueKey(t *testing.T) {+ getter, lister := buildFixedMockDatabaseMetadataGetterAndLister()++ _, err := GenerateRestoreSQL(context.Background(), base.RestoreContext{+ GetDatabaseMetadataFunc: getter,+ ListDatabaseNamesFunc: lister,+ IsCaseSensitive: false,+ }, "UPDATE test SET a = 1, c = 2 WHERE b = 3;", &store.PriorBackupDetail_Item{+ SourceTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/db",+ Table: "test",+ },+ TargetTable: &store.PriorBackupDetail_Item_Table{+ Database: "instances/i1/databases/bbarchive",+ Table: "prefix_1_test",+ },+ StartPosition: &store.Position{Line: 0, Column: 0},+ EndPosition: &store.Position{Line: math.MaxInt32, Column: 0},+ })++ require.Error(t, err, "UPDATE that touches every unique key must surface a no-disjoint-key error")+ require.Contains(t, err.Error(), "no disjoint unique key found",+ "error must be the no-disjoint-key error from generateUpdateRestore, not a different failure mode")+}diff --git a/backend/plugin/parser/tidb/test-data/test_restore.yaml b/backend/plugin/parser/tidb/test-data/test_restore.yamlnew file mode 100644index 0000000000..28ad67613c--- /dev/null+++ b/backend/plugin/parser/tidb/test-data/test_restore.yaml@@ -0,0 +1,163 @@+- input: |-+ UPDATE t_generated1111 SET a = 1 WHERE b = 1;+ UPDATE t_generated SET b = 1 WHERE a = 1;+ UPDATE t_generated SET b = 2 WHERE a = 2;+ UPDATE t_generated SET b = 3 WHERE a = 3;+ UPDATE t_generated1111 SET a = 4 WHERE b = 4;+ UPDATE t_generated SET b = 4 WHERE a = 4;+ UPDATE t_generated SET b = 5 WHERE a = 5;+ UPDATE t_generated1111 SET a = 5 WHERE b = 5;+ UPDATE t_generated SET b = 6 WHERE a = 6;+ UPDATE t_generated SET b = 6 WHERE a = 6;+ UPDATE t_generated111 SET a = 7 WHERE b = 7;+ UPDATE t_generated SET b = 7 WHERE a = 7;+ backupdatabase: bbarchive+ backuptable: prefix_t_generated+ originaldatabase: db+ originaltable: t_generated+ result: |-+ /*+ Original SQL:++ UPDATE t_generated SET b = 1 WHERE a = 1;+ UPDATE t_generated SET b = 2 WHERE a = 2;+ UPDATE t_generated SET b = 3 WHERE a = 3;+ UPDATE t_generated SET b = 4 WHERE a = 4;+ UPDATE t_generated SET b = 5 WHERE a = 5;+ UPDATE t_generated SET b = 6 WHERE a = 6;+ UPDATE t_generated SET b = 6 WHERE a = 6;+ UPDATE t_generated SET b = 7 WHERE a = 7;+ */+ INSERT INTO `db`.`t_generated` (`a`, `b`) SELECT `a`, `b` FROM `bbarchive`.`prefix_t_generated` ON DUPLICATE KEY UPDATE `b` = VALUES(`b`);+- input: |-+ UPDATE t_generated1111 SET a = 1 WHERE b = 1;+ UPDATE t_generated SET a = 1 WHERE b = 1;+ UPDATE t_generated SET a = 2 WHERE b = 2;+ UPDATE t_generated SET a = 3 WHERE b = 3;+ UPDATE t_generated1111 SET a = 4 WHERE b = 4;+ UPDATE t_generated SET a = 4 WHERE b = 4;+ UPDATE t_generated SET a = 5 WHERE b = 5;+ UPDATE t_generated1111 SET a = 5 WHERE b = 5;+ UPDATE t_generated SET a = 6 WHERE b = 6;+ UPDATE t_generated SET a = 6 WHERE b = 6;+ UPDATE t_generated111 SET a = 7 WHERE b = 7;+ UPDATE t_generated SET a = 7 WHERE b = 7;+ backupdatabase: bbarchive+ backuptable: prefix_t_generated+ originaldatabase: db+ originaltable: t_generated+ result: |-+ /*+ Original SQL:++ UPDATE t_generated SET a = 1 WHERE b = 1;+ UPDATE t_generated SET a = 2 WHERE b = 2;+ UPDATE t_generated SET a = 3 WHERE b = 3;+ UPDATE t_generated SET a = 4 WHERE b = 4;+ UPDATE t_generated SET a = 5 WHERE b = 5;+ UPDATE t_generated SET a = 6 WHERE b = 6;+ UPDATE t_generated SET a = 6 WHERE b = 6;+ UPDATE t_generated SET a = 7 WHERE b = 7;+ */+ INSERT INTO `db`.`t_generated` (`a`, `b`) SELECT `a`, `b` FROM `bbarchive`.`prefix_t_generated` ON DUPLICATE KEY UPDATE `a` = VALUES(`a`);+- input: |-+ UPDATE t_generated SET a = 1 WHERE b = 1;+ UPDATE t_generated SET a = 2 WHERE b = 2;+ UPDATE t_generated SET a = 3 WHERE b = 3;+ UPDATE t_generated SET a = 4 WHERE b = 4;+ UPDATE t_generated SET a = 5 WHERE b = 5;+ UPDATE t_generated SET a = 6 WHERE b = 6;+ UPDATE t_generated SET a = 7 WHERE b = 7;+ backupdatabase: bbarchive+ backuptable: prefix_t_generated+ originaldatabase: db+ originaltable: t_generated+ result: |-+ /*+ Original SQL:+ UPDATE t_generated SET a = 1 WHERE b = 1;+ UPDATE t_generated SET a = 2 WHERE b = 2;+ UPDATE t_generated SET a = 3 WHERE b = 3;+ UPDATE t_generated SET a = 4 WHERE b = 4;+ UPDATE t_generated SET a = 5 WHERE b = 5;+ UPDATE t_generated SET a = 6 WHERE b = 6;+ UPDATE t_generated SET a = 7 WHERE b = 7;+ */+ INSERT INTO `db`.`t_generated` (`a`, `b`) SELECT `a`, `b` FROM `bbarchive`.`prefix_t_generated` ON DUPLICATE KEY UPDATE `a` = VALUES(`a`);+- input: DELETE FROM t_generated where a = 1;+ backupdatabase: bbarchive+ backuptable: prefix_1_t_generated+ originaldatabase: db+ originaltable: t_generated+ result: |-+ /*+ Original SQL:+ DELETE FROM t_generated where a = 1;+ */+ INSERT INTO `db`.`t_generated` (`a`, `b`) SELECT `a`, `b` FROM `bbarchive`.`prefix_1_t_generated`;+- input: UPDATE t_generated SET a = 1 WHERE a = 2;+ backupdatabase: bbarchive+ backuptable: prefix_1_t_generated+ originaldatabase: db+ originaltable: t_generated+ result: |-+ /*+ Original SQL:+ UPDATE t_generated SET a = 1 WHERE a = 2;+ */+ INSERT INTO `db`.`t_generated` (`a`, `b`) SELECT `a`, `b` FROM `bbarchive`.`prefix_1_t_generated` ON DUPLICATE KEY UPDATE `a` = VALUES(`a`);+- input: DELETE test FROM test, test2 as t2 where test.id = t2.id;+ backupdatabase: bbarchive+ backuptable: prefix_1_test+ originaldatabase: db+ originaltable: test+ result: |-+ /*+ Original SQL:+ DELETE test FROM test, test2 as t2 where test.id = t2.id;+ */+ INSERT INTO `db`.`test` SELECT * FROM `bbarchive`.`prefix_1_test`;+- input: UPDATE test x SET x.c = 1 WHERE x.c = 1;+ backupdatabase: bbarchive+ backuptable: prefix_1_test+ originaldatabase: db+ originaltable: test+ result: |-+ /*+ Original SQL:+ UPDATE test x SET x.c = 1 WHERE x.c = 1;+ */+ INSERT INTO `db`.`test` SELECT * FROM `bbarchive`.`prefix_1_test` ON DUPLICATE KEY UPDATE `c` = VALUES(`c`);+- input: UPDATE test SET a = 1 WHERE c = 1;+ backupdatabase: bbarchive+ backuptable: prefix_1_test+ originaldatabase: db+ originaltable: test+ result: |-+ /*+ Original SQL:+ UPDATE test SET a = 1 WHERE c = 1;+ */+ INSERT INTO `db`.`test` SELECT * FROM `bbarchive`.`prefix_1_test` ON DUPLICATE KEY UPDATE `a` = VALUES(`a`);+- input: |-+ UPDATE test SET a = 1 WHERE c = 1;+ UPDATE test SET b = 2 WHERE c = 2;+ UPDATE test SET a = 3 WHERE c = 3;+ UPDATE test SET b = 4 WHERE c = 4;+ UPDATE test SET a = 5 WHERE c = 5;+ UPDATE test SET b = 6 WHERE c = 6;+ backupdatabase: bbarchive+ backuptable: prefix_1_test+ originaldatabase: db+ originaltable: test+ result: |-+ /*+ Original SQL:+ UPDATE test SET a = 1 WHERE c = 1;+ UPDATE test SET b = 2 WHERE c = 2;+ UPDATE test SET a = 3 WHERE c = 3;+ UPDATE test SET b = 4 WHERE c = 4;+ UPDATE test SET a = 5 WHERE c = 5;+ UPDATE test SET b = 6 WHERE c = 6;+ */+ INSERT INTO `db`.`test` SELECT * FROM `bbarchive`.`prefix_1_test` ON DUPLICATE KEY UPDATE `a` = VALUES(`a`), `b` = VALUES(`b`);diff --git a/docker/tidb.yaml b/docker/tidb.yamlnew file mode 100644index 0000000000..9a96160b8f--- /dev/null+++ b/docker/tidb.yaml@@ -0,0 +1,6 @@+services:+ tidb:+ image: pingcap/tidb:v8.5.5+ container_name: tidb-test+ ports:+ - "4000:4000"