Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(tidb): migrate dml_dry_run advisor to omni AST (#20467)

Vincent Huang0fd7efd92ab3
diff --git a/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run.go b/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run.go
index 7f4ed41590..4bf5aac248 100644
--- a/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run.go
+++ b/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run.go
@@ -1,15 +1,12 @@
package tidb
-// Framework code is generated by the generator.
-
import (
"context"
"database/sql"
"fmt"
"strings"
- "github.com/pingcap/tidb/pkg/parser/ast"
- "github.com/pingcap/tidb/pkg/parser/format"
+ "github.com/bytebase/omni/tidb/ast"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -19,7 +16,6 @@ import (
var (
_ advisor.Advisor = (*StatementDmlDryRunAdvisor)(nil)
- _ ast.Visitor = (*statementDmlDryRunChecker)(nil)
)
func init() {
@@ -32,8 +28,7 @@ type StatementDmlDryRunAdvisor struct {
// Check checks for DML dry run.
func (*StatementDmlDryRunAdvisor) Check(ctx context.Context, checkCtx advisor.Context) ([]*storepb.Advice, error) {
- stmtList, err := getTiDBNodes(checkCtx)
-
+ stmts, err := getTiDBOmniNodes(checkCtx)
if err != nil {
return nil, err
}
@@ -42,27 +37,27 @@ func (*StatementDmlDryRunAdvisor) Check(ctx context.Context, checkCtx advisor.Co
if err != nil {
return nil, err
}
+
// BYT-8855: Skip DML dry run if there are DDL statements mixed in, because DML
// statements often reference objects created by DDL statements, causing false positives.
if advisor.ContainsDDL(checkCtx.DBType, checkCtx.ParsedStatements) {
return nil, nil
}
+ if checkCtx.Driver == nil {
+ return nil, nil
+ }
+
checker := &statementDmlDryRunChecker{
+ ctx: ctx,
+ driver: checkCtx.Driver,
level: level,
title: checkCtx.Rule.Type.String(),
- driver: checkCtx.Driver,
- ctx: ctx,
}
-
- if checker.driver != nil {
- for _, stmt := range stmtList {
- checker.text = stmt.Text()
- checker.line = stmt.OriginTextPosition()
- (stmt).Accept(checker)
- if checker.explainCount >= common.MaximumLintExplainSize {
- break
- }
+ for _, ostmt := range stmts {
+ checker.check(ostmt)
+ if checker.explainCount >= common.MaximumLintExplainSize {
+ break
}
}
@@ -70,138 +65,149 @@ func (*StatementDmlDryRunAdvisor) Check(ctx context.Context, checkCtx advisor.Co
}
type statementDmlDryRunChecker struct {
- adviceList []*storepb.Advice
+ ctx context.Context
+ driver *sql.DB
level storepb.Advice_Status
title string
- text string
- line int
- driver *sql.DB
- ctx context.Context
+ adviceList []*storepb.Advice
explainCount int
}
-// Enter implements the ast.Visitor interface.
-func (checker *statementDmlDryRunChecker) Enter(in ast.Node) (ast.Node, bool) {
- switch node := in.(type) {
- case *ast.NonTransactionalDMLStmt:
- // TiDB's BATCH syntax (e.g., "BATCH ON id LIMIT 5000 UPDATE ...")
- // We need to:
- // 1. Run "BATCH ... DRY RUN" to validate the batch splitting logic
- // 2. Run "EXPLAIN" on the inner DML to validate the DML itself
- checker.explainCount++
- innerDML := node.DMLStmt
- if innerDML == nil {
- return in, false
- }
-
- // Step 1: Run TiDB's native BATCH DRY RUN to validate the batch splitting.
- // This checks that the shard column exists, the LIMIT is valid, etc.
- // We restore the NonTransactionalDMLStmt with DryRun=1 to generate "BATCH ... DRY RUN ..."
- // Note: BATCH DRY RUN requires auto-commit mode, so we use QueryContext directly
- // instead of advisor.Query which wraps in a transaction.
- batchDryRunSQL, err := checker.restoreNonTransactionalDMLWithDryRun(node)
- if err == nil && batchDryRunSQL != "" {
- if err := checker.queryInAutoCommit(batchDryRunSQL); err != nil {
- checker.adviceList = append(checker.adviceList, &storepb.Advice{
- Status: checker.level,
- Code: code.StatementDMLDryRunFailed.Int32(),
- Title: checker.title,
- Content: fmt.Sprintf("\"%s\" dry runs failed: %s", checker.text, err.Error()),
- StartPosition: common.ConvertANTLRLineToPosition(checker.line),
- })
- // Don't continue to EXPLAIN if BATCH DRY RUN failed
- return in, true
- }
- }
-
- // Step 2: Run EXPLAIN on the inner DML to validate the DML execution plan.
- // EXPLAIN doesn't support BATCH syntax, so we extract and EXPLAIN the inner DML.
- innerSQL, err := checker.restoreNode(innerDML)
- if err != nil {
- checker.adviceList = append(checker.adviceList, &storepb.Advice{
- Status: checker.level,
- Code: code.StatementDMLDryRunFailed.Int32(),
- Title: checker.title,
- Content: fmt.Sprintf("\"%s\" dry runs failed: failed to extract inner DML: %s", checker.text, err.Error()),
- StartPosition: common.ConvertANTLRLineToPosition(checker.line),
- })
- return in, true
- }
- if _, err := advisor.Query(checker.ctx, advisor.QueryContext{}, checker.driver, storepb.Engine_TIDB, fmt.Sprintf("EXPLAIN %s", innerSQL)); err != nil {
- checker.adviceList = append(checker.adviceList, &storepb.Advice{
- Status: checker.level,
- Code: code.StatementDMLDryRunFailed.Int32(),
- Title: checker.title,
- Content: fmt.Sprintf("\"%s\" dry runs failed: %s", checker.text, err.Error()),
- StartPosition: common.ConvertANTLRLineToPosition(checker.line),
- })
- }
- // Don't visit children since we've already handled the inner DML
- return in, true
+// check dispatches a single top-level statement. Recipe A (top-level
+// type-switch): omni surfaces TiDB BATCH as *ast.BatchStmt and plain DML as
+// *ast.{Insert,Update,Delete}Stmt — both top-level here, so a walk is
+// unnecessary. EXPLAIN-wrapped DML (e.g. "EXPLAIN DELETE ...") is degenerate
+// input for a dry-run advisor and intentionally not descended into.
+func (c *statementDmlDryRunChecker) check(ostmt OmniStmt) {
+ switch node := ostmt.Node.(type) {
+ case *ast.BatchStmt:
+ c.checkBatch(ostmt, node)
case *ast.InsertStmt, *ast.UpdateStmt, *ast.DeleteStmt:
- checker.explainCount++
- // Get the SQL text - use checker.text which contains the original statement text
- // node.Text() may be empty for nested nodes (e.g., DML inside NonTransactionalDMLStmt)
- sqlText := node.(ast.StmtNode).Text()
- if sqlText == "" {
- sqlText = checker.text
- }
- if _, err := advisor.Query(checker.ctx, advisor.QueryContext{}, checker.driver, storepb.Engine_TIDB, fmt.Sprintf("EXPLAIN %s", sqlText)); err != nil {
- checker.adviceList = append(checker.adviceList, &storepb.Advice{
- Status: checker.level,
- Code: code.StatementDMLDryRunFailed.Int32(),
- Title: checker.title,
- Content: fmt.Sprintf("\"%s\" dry runs failed: %s", checker.text, err.Error()),
- StartPosition: common.ConvertANTLRLineToPosition(checker.line),
- })
- }
+ c.explainCount++
+ c.explain(ostmt, strings.TrimRight(ostmt.TrimmedText(), ";"))
default:
}
-
- return in, false
}
-// restoreNode converts an AST node back to SQL text.
-func (*statementDmlDryRunChecker) restoreNode(node ast.Node) (string, error) {
- var buf strings.Builder
- ctx := format.NewRestoreCtx(format.DefaultRestoreFlags, &buf)
- if err := node.Restore(ctx); err != nil {
- return "", err
+// checkBatch validates a TiDB BATCH (non-transactional DML) statement, e.g.
+// "BATCH ON id LIMIT 5000 DELETE ...":
+// 1. Run "BATCH ... DRY RUN ..." to validate the batch-splitting logic.
+// 2. EXPLAIN the inner DML to validate the DML itself.
+func (c *statementDmlDryRunChecker) checkBatch(ostmt OmniStmt, node *ast.BatchStmt) {
+ c.explainCount++
+
+ // Step 1: validate batch splitting via TiDB's native BATCH DRY RUN. omni
+ // has no statement-level SQL deparse, so the DRY RUN form is derived by
+ // injecting the keyword into the original statement text immediately before
+ // the inner DML (located via the inner DML node's Loc). BATCH DRY RUN
+ // requires auto-commit mode, so we use QueryContext directly instead of
+ // advisor.Query (which wraps in a transaction).
+ if dryRunSQL, ok := dmlDryRunBatchSQL(ostmt, node); ok {
+ if err := c.queryInAutoCommit(dryRunSQL); err != nil {
+ c.appendFailure(ostmt, err.Error())
+ // Don't continue to EXPLAIN if BATCH DRY RUN failed.
+ return
+ }
}
- return buf.String(), nil
-}
-// restoreNonTransactionalDMLWithDryRun generates a "BATCH ... DRY RUN ..." statement
-// from a NonTransactionalDMLStmt to validate the batch splitting logic.
-func (*statementDmlDryRunChecker) restoreNonTransactionalDMLWithDryRun(node *ast.NonTransactionalDMLStmt) (string, error) {
- // Create a copy with DryRun enabled
- // DryRun values: 0 = no dry run, 1 = dry run the split DMLs, 2 = dry run the query
- nodeCopy := *node
- nodeCopy.DryRun = 1
+ // Step 2: EXPLAIN the inner DML. EXPLAIN doesn't accept BATCH syntax, so we
+ // slice the inner DML out of the original text via its Loc.
+ innerSQL, ok := dmlDryRunInnerSQL(ostmt, node)
+ if !ok {
+ c.appendFailure(ostmt, "failed to extract inner DML")
+ return
+ }
+ c.explain(ostmt, innerSQL)
+}
- var buf strings.Builder
- ctx := format.NewRestoreCtx(format.DefaultRestoreFlags, &buf)
- if err := nodeCopy.Restore(ctx); err != nil {
- return "", err
+// explain runs EXPLAIN on sql and records advice if it fails. The advice text
+// is the full original statement (ostmt), matching the pre-omni behavior where
+// BATCH advices reported the whole BATCH statement, not the inner DML.
+func (c *statementDmlDryRunChecker) explain(ostmt OmniStmt, sql string) {
+ if _, err := advisor.Query(c.ctx, advisor.QueryContext{}, c.driver, storepb.Engine_TIDB, fmt.Sprintf("EXPLAIN %s", sql)); err != nil {
+ c.appendFailure(ostmt, err.Error())
}
- return buf.String(), nil
}
-// queryInAutoCommit runs a query in auto-commit mode (without wrapping in a transaction).
-// This is required for TiDB's BATCH DRY RUN which cannot run inside a transaction.
-func (checker *statementDmlDryRunChecker) queryInAutoCommit(statement string) error {
- rows, err := checker.driver.QueryContext(checker.ctx, statement)
+func (c *statementDmlDryRunChecker) appendFailure(ostmt OmniStmt, reason string) {
+ c.adviceList = append(c.adviceList, &storepb.Advice{
+ Status: c.level,
+ Code: code.StatementDMLDryRunFailed.Int32(),
+ Title: c.title,
+ Content: fmt.Sprintf("\"%s\" dry runs failed: %s", ostmt.TrimmedText(), reason),
+ StartPosition: common.ConvertANTLRLineToPosition(ostmt.FirstTokenLine()),
+ })
+}
+
+// queryInAutoCommit runs a query in auto-commit mode (without wrapping in a
+// transaction). Required for TiDB's BATCH DRY RUN, which cannot run inside a
+// transaction.
+func (c *statementDmlDryRunChecker) queryInAutoCommit(statement string) error {
+ rows, err := c.driver.QueryContext(c.ctx, statement)
if err != nil {
return err
}
defer rows.Close()
- // Drain the rows to ensure the query completes
+ // Drain the rows to ensure the query completes.
for rows.Next() {
}
return rows.Err()
}
-// Leave implements the ast.Visitor interface.
-func (*statementDmlDryRunChecker) Leave(in ast.Node) (ast.Node, bool) {
- return in, true
+// dmlDryRunBatchSQL returns a "BATCH ... DRY RUN ..." statement for node.
+//
+// omni has no statement-level SQL deparse, so the DRY RUN form is derived by
+// injecting "DRY RUN " before the inner DML keyword (located via the inner DML
+// node's Loc), preserving the user's original ON/LIMIT clause, comments, and
+// whitespace verbatim — more faithful than a deparser.
+//
+// "DRY RUN" selects TiDB's split-DML dry run (pingcap DryRunSplitDml=2), which
+// validates the generated split DMLs — the batch-splitting logic this advisor
+// exists to check. The pre-omni advisor set DryRun=1, i.e. pingcap's
+// DryRunQuery ("DRY RUN QUERY"), which validates only the shard-splitting
+// SELECT, not the split DMLs — contradicting its own comment. Container-
+// verified: "DRY RUN QUERY" passes a bad inner-DML WHERE that "DRY RUN" rejects
+// (the advisor's EXPLAIN step also rejects it, so the net advice is unchanged).
+//
+// If the source already specifies a dry-run mode, it is run as-is; injecting
+// again would yield "DRY RUN DRY RUN".
+func dmlDryRunBatchSQL(ostmt OmniStmt, node *ast.BatchStmt) (string, bool) {
+ if node.DryRun != ast.BatchDryRunNone {
+ return ostmt.Text, true
+ }
+ loc, ok := dmlDryRunInnerLoc(node.DML)
+ if !ok || loc.Start <= 0 || loc.Start > len(ostmt.Text) {
+ return "", false
+ }
+ return ostmt.Text[:loc.Start] + "DRY RUN " + ostmt.Text[loc.Start:], true
+}
+
+// dmlDryRunInnerSQL slices the inner DML out of the original BATCH statement
+// text via the inner DML node's Loc (statement-relative byte offsets).
+func dmlDryRunInnerSQL(ostmt OmniStmt, node *ast.BatchStmt) (string, bool) {
+ loc, ok := dmlDryRunInnerLoc(node.DML)
+ if !ok || loc.Start <= 0 || loc.Start > len(ostmt.Text) {
+ return "", false
+ }
+ end := loc.End
+ if end <= loc.Start || end > len(ostmt.Text) {
+ end = len(ostmt.Text)
+ }
+ return strings.TrimRight(strings.TrimSpace(ostmt.Text[loc.Start:end]), ";"), true
+}
+
+// dmlDryRunInnerLoc returns the source location of a BATCH inner DML node. The
+// field is interface-typed (ast.StmtNode), concretely one of the three DML
+// node types omni produces under BATCH (REPLACE is *InsertStmt{IsReplace}).
+func dmlDryRunInnerLoc(n ast.StmtNode) (ast.Loc, bool) {
+ switch s := n.(type) {
+ case *ast.DeleteStmt:
+ return s.Loc, true
+ case *ast.UpdateStmt:
+ return s.Loc, true
+ case *ast.InsertStmt:
+ return s.Loc, true
+ default:
+ return ast.Loc{}, false
+ }
}
diff --git a/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run_testcontainer_test.go b/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run_testcontainer_test.go
new file mode 100644
index 0000000000..c457bdba41
--- /dev/null
+++ b/backend/plugin/advisor/tidb/advisor_statement_dml_dry_run_testcontainer_test.go
@@ -0,0 +1,101 @@
+package tidb
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "testing"
+
+ _ "github.com/go-sql-driver/mysql"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ "github.com/bytebase/bytebase/backend/component/sheet"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/advisor"
+ "github.com/bytebase/bytebase/backend/plugin/advisor/code"
+)
+
+// TestStatementDMLDryRunTiDBContainer exercises the dml_dry_run advisor against
+// a real TiDB instance. The fixture-based TestTiDBRules path runs with a nil
+// Driver, so it never reaches the EXPLAIN / BATCH DRY RUN logic — this is the
+// only test that validates the live behavior after the omni migration (B1.2),
+// in particular that the BATCH DRY RUN and inner-DML SQL derived by text-slicing
+// (omni has no statement-level deparse) execute correctly against TiDB.
+func TestStatementDMLDryRunTiDBContainer(t *testing.T) {
+ if testing.Short() {
+ t.Skip("Skipping TiDB testcontainer test in short mode")
+ }
+ ctx := context.Background()
+ container := testcontainer.GetTestTiDBContainer(ctx, t)
+ defer container.Close(ctx)
+
+ const database = "dml_dry_run_test"
+ if _, err := container.GetDB().ExecContext(ctx, "CREATE DATABASE IF NOT EXISTS "+database); err != nil {
+ t.Fatalf("create database: %v", err)
+ }
+
+ dsn := fmt.Sprintf("root@tcp(%s:%s)/%s?multiStatements=true&tls=false", container.GetHost(), container.GetPort(), database)
+ db, err := sql.Open("mysql", dsn)
+ require.NoError(t, err)
+ defer db.Close()
+ for _, ddl := range []string{
+ "CREATE TABLE t (id INT PRIMARY KEY, name VARCHAR(50))",
+ "CREATE TABLE t2 (id INT PRIMARY KEY, name VARCHAR(50))",
+ "INSERT INTO t VALUES (1, 'a'), (2, 'b'), (3, 'c')",
+ } {
+ _, err := db.ExecContext(ctx, ddl)
+ require.NoError(t, err, ddl)
+ }
+
+ sm := sheet.NewManager()
+ rule := &storepb.SQLReviewRule{Type: storepb.SQLReviewRule_STATEMENT_DML_DRY_RUN, Level: storepb.SQLReviewRule_WARNING}
+
+ testCases := []struct {
+ name string
+ statement string
+ wantAdvice bool
+ }{
+ {"plain delete valid", "DELETE FROM t WHERE id = 1", false},
+ {"plain update valid", "UPDATE t SET name = 'x' WHERE id = 1", false},
+ {"plain insert valid", "INSERT INTO t VALUES (4, 'd')", false},
+ {"plain unknown column", "DELETE FROM t WHERE missing_col = 1", true},
+ {"plain unknown table", "DELETE FROM missing_table WHERE id = 1", true},
+ {"batch delete valid", "BATCH ON id LIMIT 2 DELETE FROM t WHERE id > 0", false},
+ {"batch update valid", "BATCH ON id LIMIT 2 UPDATE t SET name = 'y' WHERE id > 0", false},
+ {"batch insert-select valid", "BATCH ON id LIMIT 2 INSERT INTO t2 SELECT id, name FROM t", false},
+ {"batch no-on valid", "BATCH LIMIT 2 DELETE FROM t WHERE id > 0", false},
+ {"batch inner unknown column", "BATCH ON id LIMIT 2 DELETE FROM t WHERE missing_col > 0", true},
+ {"batch unknown shard column", "BATCH ON missing_col LIMIT 2 DELETE FROM t WHERE id > 0", true},
+ // TiDB rejects non-transactional REPLACE ... VALUES (only INSERT ... SELECT
+ // is supported). The advisor must surface that as a dry-run failure; this
+ // also exercises the REPLACE (*InsertStmt{IsReplace}) text-slice path.
+ {"batch replace-values rejected by tidb", "BATCH ON id LIMIT 2 REPLACE INTO t VALUES (1, 'z')", true},
+ }
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ checkCtx := advisor.Context{
+ DBType: storepb.Engine_TIDB,
+ Driver: db,
+ CurrentDatabase: database,
+ NoAppendBuiltin: true,
+ }
+ advices, err := advisor.SQLReviewCheck(ctx, sm, tc.statement, []*storepb.SQLReviewRule{rule}, checkCtx)
+ require.NoError(t, err)
+ if tc.wantAdvice {
+ require.NotEmpty(t, advices, "expected dry-run advice for %q", tc.statement)
+ for _, a := range advices {
+ require.Equal(t, code.StatementDMLDryRunFailed.Int32(), a.Code)
+ }
+ } else {
+ require.Empty(t, advices, "expected no advice for %q", tc.statement)
+ }
+ })
+ }
+
+ // A dry run must not mutate data: all 3 seed rows remain (no test case INSERTs
+ // persist, since EXPLAIN / BATCH DRY RUN never execute the DML).
+ var count int
+ require.NoError(t, db.QueryRowContext(ctx, "SELECT COUNT(*) FROM t").Scan(&count))
+ require.Equal(t, 3, count, "dry run must not mutate data")
+}