Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(tidb): migrate dml_dry_run advisor to omni AST (#20467)
Vincent Huang0fd7efd92ab3
diff --git a/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run.go b/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run.goindex 7f4ed41590..4bf5aac248 100644--- a/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run.go+++ b/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run.go@@ -1,15 +1,12 @@package tidb-// Framework code is generated by the generator.-import ("context""database/sql""fmt""strings"- "github.com/pingcap/tidb/pkg/parser/ast"- "github.com/pingcap/tidb/pkg/parser/format"+ "github.com/bytebase/omni/tidb/ast""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -19,7 +16,6 @@ import (var (_ advisor.Advisor = (*StatementDmlDryRunAdvisor)(nil)- _ ast.Visitor = (*statementDmlDryRunChecker)(nil))func init() {@@ -32,8 +28,7 @@ type StatementDmlDryRunAdvisor struct {// Check checks for DML dry run.func (*StatementDmlDryRunAdvisor) Check(ctx context.Context, checkCtx advisor.Context) ([]*storepb.Advice, error) {- stmtList, err := getTiDBNodes(checkCtx)-+ stmts, err := getTiDBOmniNodes(checkCtx)if err != nil {return nil, err}@@ -42,27 +37,27 @@ func (*StatementDmlDryRunAdvisor) Check(ctx context.Context, checkCtx advisor.Coif err != nil {return nil, err}+// BYT-8855: Skip DML dry run if there are DDL statements mixed in, because DML// statements often reference objects created by DDL statements, causing false positives.if advisor.ContainsDDL(checkCtx.DBType, checkCtx.ParsedStatements) {return nil, nil}+ if checkCtx.Driver == nil {+ return nil, nil+ }+checker := &statementDmlDryRunChecker{+ ctx: ctx,+ driver: checkCtx.Driver,level: level,title: checkCtx.Rule.Type.String(),- driver: checkCtx.Driver,- ctx: ctx,}-- if checker.driver != nil {- for _, stmt := range stmtList {- checker.text = stmt.Text()- checker.line = stmt.OriginTextPosition()- (stmt).Accept(checker)- if checker.explainCount >= common.MaximumLintExplainSize {- break- }+ for _, ostmt := range stmts {+ checker.check(ostmt)+ if checker.explainCount >= common.MaximumLintExplainSize {+ break}}@@ -70,138 +65,149 @@ func (*StatementDmlDryRunAdvisor) Check(ctx context.Context, checkCtx advisor.Co}type statementDmlDryRunChecker struct {- adviceList []*storepb.Advice+ ctx context.Context+ driver *sql.DBlevel storepb.Advice_Statustitle string- text string- line int- driver *sql.DB- ctx context.Context+ adviceList []*storepb.AdviceexplainCount int}-// Enter implements the ast.Visitor interface.-func (checker *statementDmlDryRunChecker) Enter(in ast.Node) (ast.Node, bool) {- switch node := in.(type) {- case *ast.NonTransactionalDMLStmt:- // TiDB's BATCH syntax (e.g., "BATCH ON id LIMIT 5000 UPDATE ...")- // We need to:- // 1. Run "BATCH ... DRY RUN" to validate the batch splitting logic- // 2. Run "EXPLAIN" on the inner DML to validate the DML itself- checker.explainCount++- innerDML := node.DMLStmt- if innerDML == nil {- return in, false- }-- // Step 1: Run TiDB's native BATCH DRY RUN to validate the batch splitting.- // This checks that the shard column exists, the LIMIT is valid, etc.- // We restore the NonTransactionalDMLStmt with DryRun=1 to generate "BATCH ... DRY RUN ..."- // Note: BATCH DRY RUN requires auto-commit mode, so we use QueryContext directly- // instead of advisor.Query which wraps in a transaction.- batchDryRunSQL, err := checker.restoreNonTransactionalDMLWithDryRun(node)- if err == nil && batchDryRunSQL != "" {- if err := checker.queryInAutoCommit(batchDryRunSQL); err != nil {- checker.adviceList = append(checker.adviceList, &storepb.Advice{- Status: checker.level,- Code: code.StatementDMLDryRunFailed.Int32(),- Title: checker.title,- Content: fmt.Sprintf("\"%s\" dry runs failed: %s", checker.text, err.Error()),- StartPosition: common.ConvertANTLRLineToPosition(checker.line),- })- // Don't continue to EXPLAIN if BATCH DRY RUN failed- return in, true- }- }-- // Step 2: Run EXPLAIN on the inner DML to validate the DML execution plan.- // EXPLAIN doesn't support BATCH syntax, so we extract and EXPLAIN the inner DML.- innerSQL, err := checker.restoreNode(innerDML)- if err != nil {- checker.adviceList = append(checker.adviceList, &storepb.Advice{- Status: checker.level,- Code: code.StatementDMLDryRunFailed.Int32(),- Title: checker.title,- Content: fmt.Sprintf("\"%s\" dry runs failed: failed to extract inner DML: %s", checker.text, err.Error()),- StartPosition: common.ConvertANTLRLineToPosition(checker.line),- })- return in, true- }- if _, err := advisor.Query(checker.ctx, advisor.QueryContext{}, checker.driver, storepb.Engine_TIDB, fmt.Sprintf("EXPLAIN %s", innerSQL)); err != nil {- checker.adviceList = append(checker.adviceList, &storepb.Advice{- Status: checker.level,- Code: code.StatementDMLDryRunFailed.Int32(),- Title: checker.title,- Content: fmt.Sprintf("\"%s\" dry runs failed: %s", checker.text, err.Error()),- StartPosition: common.ConvertANTLRLineToPosition(checker.line),- })- }- // Don't visit children since we've already handled the inner DML- return in, true+// check dispatches a single top-level statement. Recipe A (top-level+// type-switch): omni surfaces TiDB BATCH as *ast.BatchStmt and plain DML as+// *ast.{Insert,Update,Delete}Stmt — both top-level here, so a walk is+// unnecessary. EXPLAIN-wrapped DML (e.g. "EXPLAIN DELETE ...") is degenerate+// input for a dry-run advisor and intentionally not descended into.+func (c *statementDmlDryRunChecker) check(ostmt OmniStmt) {+ switch node := ostmt.Node.(type) {+ case *ast.BatchStmt:+ c.checkBatch(ostmt, node)case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt:- checker.explainCount++- // Get the SQL text - use checker.text which contains the original statement text- // node.Text() may be empty for nested nodes (e.g., DML inside NonTransactionalDMLStmt)- sqlText := node.(ast.StmtNode).Text()- if sqlText == "" {- sqlText = checker.text- }- if _, err := advisor.Query(checker.ctx, advisor.QueryContext{}, checker.driver, storepb.Engine_TIDB, fmt.Sprintf("EXPLAIN %s", sqlText)); err != nil {- checker.adviceList = append(checker.adviceList, &storepb.Advice{- Status: checker.level,- Code: code.StatementDMLDryRunFailed.Int32(),- Title: checker.title,- Content: fmt.Sprintf("\"%s\" dry runs failed: %s", checker.text, err.Error()),- StartPosition: common.ConvertANTLRLineToPosition(checker.line),- })- }+ c.explainCount+++ c.explain(ostmt, strings.TrimRight(ostmt.TrimmedText(), ";"))default:}-- return in, false}-// restoreNode converts an AST node back to SQL text.-func (*statementDmlDryRunChecker) restoreNode(node ast.Node) (string, error) {- var buf strings.Builder- ctx := format.NewRestoreCtx(format.DefaultRestoreFlags, &buf)- if err := node.Restore(ctx); err != nil {- return "", err+// checkBatch validates a TiDB BATCH (non-transactional DML) statement, e.g.+// "BATCH ON id LIMIT 5000 DELETE ...":+// 1. Run "BATCH ... DRY RUN ..." to validate the batch-splitting logic.+// 2. EXPLAIN the inner DML to validate the DML itself.+func (c *statementDmlDryRunChecker) checkBatch(ostmt OmniStmt, node *ast.BatchStmt) {+ c.explainCount++++ // Step 1: validate batch splitting via TiDB's native BATCH DRY RUN. omni+ // has no statement-level SQL deparse, so the DRY RUN form is derived by+ // injecting the keyword into the original statement text immediately before+ // the inner DML (located via the inner DML node's Loc). BATCH DRY RUN+ // requires auto-commit mode, so we use QueryContext directly instead of+ // advisor.Query (which wraps in a transaction).+ if dryRunSQL, ok := dmlDryRunBatchSQL(ostmt, node); ok {+ if err := c.queryInAutoCommit(dryRunSQL); err != nil {+ c.appendFailure(ostmt, err.Error())+ // Don't continue to EXPLAIN if BATCH DRY RUN failed.+ return+ }}- return buf.String(), nil-}-// restoreNonTransactionalDMLWithDryRun generates a "BATCH ... DRY RUN ..." statement-// from a NonTransactionalDMLStmt to validate the batch splitting logic.-func (*statementDmlDryRunChecker) restoreNonTransactionalDMLWithDryRun(node *ast.NonTransactionalDMLStmt) (string, error) {- // Create a copy with DryRun enabled- // DryRun values: 0 = no dry run, 1 = dry run the split DMLs, 2 = dry run the query- nodeCopy := *node- nodeCopy.DryRun = 1+ // Step 2: EXPLAIN the inner DML. EXPLAIN doesn't accept BATCH syntax, so we+ // slice the inner DML out of the original text via its Loc.+ innerSQL, ok := dmlDryRunInnerSQL(ostmt, node)+ if !ok {+ c.appendFailure(ostmt, "failed to extract inner DML")+ return+ }+ c.explain(ostmt, innerSQL)+}- var buf strings.Builder- ctx := format.NewRestoreCtx(format.DefaultRestoreFlags, &buf)- if err := nodeCopy.Restore(ctx); err != nil {- return "", err+// explain runs EXPLAIN on sql and records advice if it fails. The advice text+// is the full original statement (ostmt), matching the pre-omni behavior where+// BATCH advices reported the whole BATCH statement, not the inner DML.+func (c *statementDmlDryRunChecker) explain(ostmt OmniStmt, sql string) {+ if _, err := advisor.Query(c.ctx, advisor.QueryContext{}, c.driver, storepb.Engine_TIDB, fmt.Sprintf("EXPLAIN %s", sql)); err != nil {+ c.appendFailure(ostmt, err.Error())}- return buf.String(), nil}-// queryInAutoCommit runs a query in auto-commit mode (without wrapping in a transaction).-// This is required for TiDB's BATCH DRY RUN which cannot run inside a transaction.-func (checker *statementDmlDryRunChecker) queryInAutoCommit(statement string) error {- rows, err := checker.driver.QueryContext(checker.ctx, statement)+func (c *statementDmlDryRunChecker) appendFailure(ostmt OmniStmt, reason string) {+ c.adviceList = append(c.adviceList, &storepb.Advice{+ Status: c.level,+ Code: code.StatementDMLDryRunFailed.Int32(),+ Title: c.title,+ Content: fmt.Sprintf("\"%s\" dry runs failed: %s", ostmt.TrimmedText(), reason),+ StartPosition: common.ConvertANTLRLineToPosition(ostmt.FirstTokenLine()),+ })+}++// queryInAutoCommit runs a query in auto-commit mode (without wrapping in a+// transaction). Required for TiDB's BATCH DRY RUN, which cannot run inside a+// transaction.+func (c *statementDmlDryRunChecker) queryInAutoCommit(statement string) error {+ rows, err := c.driver.QueryContext(c.ctx, statement)if err != nil {return err}defer rows.Close()- // Drain the rows to ensure the query completes+ // Drain the rows to ensure the query completes.for rows.Next() {}return rows.Err()}-// Leave implements the ast.Visitor interface.-func (*statementDmlDryRunChecker) Leave(in ast.Node) (ast.Node, bool) {- return in, true+// dmlDryRunBatchSQL returns a "BATCH ... DRY RUN ..." statement for node.+//+// omni has no statement-level SQL deparse, so the DRY RUN form is derived by+// injecting "DRY RUN " before the inner DML keyword (located via the inner DML+// node's Loc), preserving the user's original ON/LIMIT clause, comments, and+// whitespace verbatim — more faithful than a deparser.+//+// "DRY RUN" selects TiDB's split-DML dry run (pingcap DryRunSplitDml=2), which+// validates the generated split DMLs — the batch-splitting logic this advisor+// exists to check. The pre-omni advisor set DryRun=1, i.e. pingcap's+// DryRunQuery ("DRY RUN QUERY"), which validates only the shard-splitting+// SELECT, not the split DMLs — contradicting its own comment. Container-+// verified: "DRY RUN QUERY" passes a bad inner-DML WHERE that "DRY RUN" rejects+// (the advisor's EXPLAIN step also rejects it, so the net advice is unchanged).+//+// If the source already specifies a dry-run mode, it is run as-is; injecting+// again would yield "DRY RUN DRY RUN".+func dmlDryRunBatchSQL(ostmt OmniStmt, node *ast.BatchStmt) (string, bool) {+ if node.DryRun != ast.BatchDryRunNone {+ return ostmt.Text, true+ }+ loc, ok := dmlDryRunInnerLoc(node.DML)+ if !ok || loc.Start <= 0 || loc.Start > len(ostmt.Text) {+ return "", false+ }+ return ostmt.Text[:loc.Start] + "DRY RUN " + ostmt.Text[loc.Start:], true+}++// dmlDryRunInnerSQL slices the inner DML out of the original BATCH statement+// text via the inner DML node's Loc (statement-relative byte offsets).+func dmlDryRunInnerSQL(ostmt OmniStmt, node *ast.BatchStmt) (string, bool) {+ loc, ok := dmlDryRunInnerLoc(node.DML)+ if !ok || loc.Start <= 0 || loc.Start > len(ostmt.Text) {+ return "", false+ }+ end := loc.End+ if end <= loc.Start || end > len(ostmt.Text) {+ end = len(ostmt.Text)+ }+ return strings.TrimRight(strings.TrimSpace(ostmt.Text[loc.Start:end]), ";"), true+}++// dmlDryRunInnerLoc returns the source location of a BATCH inner DML node. The+// field is interface-typed (ast.StmtNode), concretely one of the three DML+// node types omni produces under BATCH (REPLACE is *InsertStmt{IsReplace}).+func dmlDryRunInnerLoc(n ast.StmtNode) (ast.Loc, bool) {+ switch s := n.(type) {+ case *ast.DeleteStmt:+ return s.Loc, true+ case *ast.UpdateStmt:+ return s.Loc, true+ case *ast.InsertStmt:+ return s.Loc, true+ default:+ return ast.Loc{}, false+ }}diff --git a/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run_testcontainer_test.go b/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run_testcontainer_test.gonew file mode 100644index 0000000000..c457bdba41--- /dev/null+++ b/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run_testcontainer_test.go@@ -0,0 +1,101 @@+package tidb++import (+ "context"+ "database/sql"+ "fmt"+ "testing"++ _ "github.com/go-sql-driver/mysql"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ "github.com/bytebase/bytebase/backend/component/sheet"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/advisor"+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"+)++// TestStatementDMLDryRunTiDBContainer exercises the dml_dry_run advisor against+// a real TiDB instance. The fixture-based TestTiDBRules path runs with a nil+// Driver, so it never reaches the EXPLAIN / BATCH DRY RUN logic — this is the+// only test that validates the live behavior after the omni migration (B1.2),+// in particular that the BATCH DRY RUN and inner-DML SQL derived by text-slicing+// (omni has no statement-level deparse) execute correctly against TiDB.+func TestStatementDMLDryRunTiDBContainer(t *testing.T) {+ if testing.Short() {+ t.Skip("Skipping TiDB testcontainer test in short mode")+ }+ ctx := context.Background()+ container := testcontainer.GetTestTiDBContainer(ctx, t)+ defer container.Close(ctx)++ const database = "dml_dry_run_test"+ if _, err := container.GetDB().ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS "+database); err != nil {+ t.Fatalf("create database: %v", err)+ }++ dsn := fmt.Sprintf("root@tcp(%s:%s)/%s?multiStatements=true&tls=false", container.GetHost(), container.GetPort(), database)+ db, err := sql.Open("mysql", dsn)+ require.NoError(t, err)+ defer db.Close()+ for _, ddl := range []string{+ "CREATE TABLE t (id INT PRIMARY KEY, name VARCHAR(50))",+ "CREATE TABLE t2 (id INT PRIMARY KEY, name VARCHAR(50))",+ "INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'c')",+ } {+ _, err := db.ExecContext(ctx, ddl)+ require.NoError(t, err, ddl)+ }++ sm := sheet.NewManager()+ rule := &storepb.SQLReviewRule{Type: storepb.SQLReviewRule_STATEMENT_DML_DRY_RUN, Level: storepb.SQLReviewRule_WARNING}++ testCases := []struct {+ name string+ statement string+ wantAdvice bool+ }{+ {"plain delete valid", "DELETE FROM t WHERE id = 1", false},+ {"plain update valid", "UPDATE t SET name = 'x' WHERE id = 1", false},+ {"plain insert valid", "INSERT INTO t VALUES (4, 'd')", false},+ {"plain unknown column", "DELETE FROM t WHERE missing_col = 1", true},+ {"plain unknown table", "DELETE FROM missing_table WHERE id = 1", true},+ {"batch delete valid", "BATCH ON id LIMIT 2 DELETE FROM t WHERE id > 0", false},+ {"batch update valid", "BATCH ON id LIMIT 2 UPDATE t SET name = 'y' WHERE id > 0", false},+ {"batch insert-select valid", "BATCH ON id LIMIT 2 INSERT INTO t2 SELECT id, name FROM t", false},+ {"batch no-on valid", "BATCH LIMIT 2 DELETE FROM t WHERE id > 0", false},+ {"batch inner unknown column", "BATCH ON id LIMIT 2 DELETE FROM t WHERE missing_col > 0", true},+ {"batch unknown shard column", "BATCH ON missing_col LIMIT 2 DELETE FROM t WHERE id > 0", true},+ // TiDB rejects non-transactional REPLACE ... VALUES (only INSERT ... SELECT+ // is supported). The advisor must surface that as a dry-run failure; this+ // also exercises the REPLACE (*InsertStmt{IsReplace}) text-slice path.+ {"batch replace-values rejected by tidb", "BATCH ON id LIMIT 2 REPLACE INTO t VALUES (1, 'z')", true},+ }+ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ checkCtx := advisor.Context{+ DBType: storepb.Engine_TIDB,+ Driver: db,+ CurrentDatabase: database,+ NoAppendBuiltin: true,+ }+ advices, err := advisor.SQLReviewCheck(ctx, sm, tc.statement, []*storepb.SQLReviewRule{rule}, checkCtx)+ require.NoError(t, err)+ if tc.wantAdvice {+ require.NotEmpty(t, advices, "expected dry-run advice for %q", tc.statement)+ for _, a := range advices {+ require.Equal(t, code.StatementDMLDryRunFailed.Int32(), a.Code)+ }+ } else {+ require.Empty(t, advices, "expected no advice for %q", tc.statement)+ }+ })+ }++ // A dry run must not mutate data: all 3 seed rows remain (no test case INSERTs+ // persist, since EXPLAIN / BATCH DRY RUN never execute the DML).+ var count int+ require.NoError(t, db.QueryRowContext(ctx, "SELECT COUNT(*) FROM t").Scan(&count))+ require.Equal(t, 3, count, "dry run must not mutate data")+}