Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(tidb): add AsPingCapAST bridge on OmniAST (Phase 1.5 §1.5.2) (#20179)
Vincent Huanga70631b8acd7
diff --git a/backend/plugin/advisor/tidb/utils.go b/backend/plugin/advisor/tidb/utils.goindex 0ad059e531..482d74de83 100644--- a/backend/plugin/advisor/tidb/utils.go+++ b/backend/plugin/advisor/tidb/utils.go@@ -84,6 +84,18 @@ func needDefault(column *ast.ColumnDef) bool {}// getTiDBNodes extracts TiDB AST nodes from the advisor context.+//+// Soft-fail on bridge miss: when stmt.AST is a PingCapASTProvider+// (post-dispatcher-flip *OmniAST) whose AsPingCapAST() returns (nil, false),+// the statement is skipped rather than surfaced as an error. This honors+// Phase 1.5 invariant #2 from the omni-tidb completion plan — un-migrated+// advisors emit no advice for the skipped statement; the review continues.+// The bridge has already logged the parse failure at debug level+// (omni.go:AsPingCapAST).+//+// True engine-mismatch (stmt.AST is some unrelated type, neither *tidb.AST+// nor a PingCapASTProvider) is still surfaced as an error — that's a+// programmer error in registration, not a soft-fail case.func getTiDBNodes(checkCtx advisor.Context) ([]ast.StmtNode, error) {if checkCtx.ParsedStatements == nil {return nil, errors.New("ParsedStatements is not provided in context")@@ -96,6 +108,13 @@ func getTiDBNodes(checkCtx advisor.Context) ([]ast.StmtNode, error) {}tidbAST, ok := tidbparser.GetTiDBAST(stmt.AST)if !ok {+ // Bridge miss → soft-fail (skip). Engine mismatch → error.+ // PingCapASTProvider implementations route through GetTiDBAST →+ // AsPingCapAST; a (nil, false) here means the bridge tried and+ // pingcap rejected the statement. Skip rather than abort the rule.+ if _, isProvider := stmt.AST.(tidbparser.PingCapASTProvider); isProvider {+ continue+ }return nil, errors.New("AST type mismatch: expected TiDB parser result")}stmtNodes = append(stmtNodes, tidbAST.Node)diff --git a/backend/plugin/advisor/tidb/utils_test.go b/backend/plugin/advisor/tidb/utils_test.goindex a646cf7e24..326f923c18 100644--- a/backend/plugin/advisor/tidb/utils_test.go+++ b/backend/plugin/advisor/tidb/utils_test.go@@ -8,6 +8,7 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/advisor""github.com/bytebase/bytebase/backend/plugin/parser/base"+ tidbparser "github.com/bytebase/bytebase/backend/plugin/parser/tidb")// TestGetTiDBOmniNodesCachesAcrossRules pins the Phase 1.5 single-parse-per-@@ -83,3 +84,52 @@ func TestGetTiDBOmniNodesSoftFailsOnGrammarGap(t *testing.T) {require.GreaterOrEqual(t, len(got), 2,"expected at least the two non-BATCH statements to parse and be returned")}++// TestGetTiDBNodesSoftFailsOnBridgeMiss pins the post-flip soft-fail+// contract on the un-migrated-advisor side: when an OmniAST's+// AsPingCapAST() returns (nil, false) — i.e. omni accepted the statement+// but pingcap rejected the same text — getTiDBNodes must SKIP that+// statement, not abort the rule with "AST type mismatch".+//+// Codex caught this on PR #20179: without the soft-fail, post-flip every+// un-migrated advisor emits "Rule check failed: AST type mismatch" for+// statements omni accepts but pingcap doesn't, breaking the migration+// continuity the bridge is meant to preserve.+func TestGetTiDBNodesSoftFailsOnBridgeMiss(t *testing.T) {+ // Pingcap rejects this; omni doesn't matter for this test — what+ // matters is that the bridge returns (nil, false) and getTiDBNodes+ // must skip rather than error.+ bridgeMiss := &tidbparser.OmniAST{+ Text: "SELECT FROM WHERE;", // pingcap-invalid+ StartPosition: &storepb.Position{Line: 1},+ }+ bridgeHit := &tidbparser.OmniAST{+ Text: "SELECT 1",+ StartPosition: &storepb.Position{Line: 2},+ }++ ctx := advisor.Context{+ ParsedStatements: []base.ParsedStatement{+ {+ Statement: base.Statement{+ Text: "SELECT FROM WHERE;",+ Start: &storepb.Position{Line: 1},+ },+ AST: bridgeMiss,+ },+ {+ Statement: base.Statement{+ Text: "SELECT 1",+ Start: &storepb.Position{Line: 2},+ },+ AST: bridgeHit,+ },+ },+ }++ got, err := getTiDBNodes(ctx)+ require.NoError(t, err,+ "bridge miss must be soft-failed (skip), not surfaced as 'AST type mismatch' — see Phase 1.5 invariant #2")+ require.Len(t, got, 1,+ "expected 1 statement from the bridge-hit branch; the bridge-miss statement should be skipped silently")+}diff --git a/backend/plugin/parser/tidb/ast.go b/backend/plugin/parser/tidb/ast.goindex 12811f66de..2f5b69d8a1 100644--- a/backend/plugin/parser/tidb/ast.go+++ b/backend/plugin/parser/tidb/ast.go@@ -20,12 +20,31 @@ func (a *AST) ASTStartPosition() *storepb.Position {return a.StartPosition}+// PingCapASTProvider is an optional interface that AST implementations can+// provide to expose an underlying native pingcap-parser AST. Used during the+// Phase 1.5 advisor migration: when the registered ParseStatementsFunc is+// flipped to omni, *OmniAST values reach un-migrated callers of GetTiDBAST,+// and those callers fall through to AsPingCapAST() to keep working.+//+// Mirrors backend/plugin/parser/base/AntlrASTProvider for the mysql migration.+// See plans/2026-04-23-omni-tidb-completion-plan.md §1.5.0 invariant #4.+type PingCapASTProvider interface {+ AsPingCapAST() (*AST, bool)+}+// GetTiDBAST extracts the TiDB AST from a base.AST.// Returns the AST and true if it is a TiDB AST, nil and false otherwise.+// Also checks for PingCapASTProvider implementations (e.g. *OmniAST during+// the advisor migration), which lazily build a native pingcap AST on demand.func GetTiDBAST(a base.AST) (*AST, bool) {if a == nil {return nil, false}- tidbAST, ok := a.(*AST)- return tidbAST, ok+ if tidbAST, ok := a.(*AST); ok {+ return tidbAST, true+ }+ if provider, ok := a.(PingCapASTProvider); ok {+ return provider.AsPingCapAST()+ }+ return nil, false}diff --git a/backend/plugin/parser/tidb/omni.go b/backend/plugin/parser/tidb/omni.goindex 5ae031b5e8..0e556bb04f 100644--- a/backend/plugin/parser/tidb/omni.go+++ b/backend/plugin/parser/tidb/omni.go@@ -1,6 +1,7 @@package tidbimport (+ "log/slog""unicode/utf8""github.com/bytebase/omni/tidb/ast"@@ -12,20 +13,22 @@ import (// OmniAST wraps an omni/tidb AST node and implements the base.AST interface.//-// Unlike mysql/OmniAST, this type does NOT implement base.AntlrASTProvider-// (AsANTLRAST) nor a hypothetical native-pingcap fallback. Rationale:-// - TiDB has no TiDB-specific ANTLR grammar; tidb/backup.go uses the mysql-// ANTLR grammar on its own text and never consumes *OmniAST.-// - The 51 advisors under backend/plugin/advisor/tidb/ use getTiDBNodes() ->-// tidbparser.GetTiDBAST(stmt.AST) which asserts to the native *tidb.AST,-// not *OmniAST. So no legacy consumer sees *OmniAST today.-// - Task 1.2 (tidb.go extension) keeps the registered ParseStatementsFunc-// pointed at the native pingcap parser ("no behavioral change yet"), so-// *OmniAST won't reach advisors until a post-Phase 1 migration.+// During the Phase 1.5 advisor migration, OmniAST also implements+// PingCapASTProvider so that callers still using GetTiDBAST() (~50 of 51+// un-migrated advisors at any given point in the migration) can fall back to+// a native pingcap-parsed AST. After the dispatcher flip (§1.5.N+1) returns+// *OmniAST from ParseStatements, those un-migrated advisors keep working+// because GetTiDBAST routes through AsPingCapAST() automatically.//-// When that migration flips the default and the 51 advisors need a compat-// path, add AsPingCapAST() + a matching provider interface in base/ast.go-// alongside the migration PR — with real consumers to exercise it.+// Bridge cleanup is deferred until Phase 2 §Tier 4g (NonTransactionalDMLStmt+// + production-grade restore equivalent) ships and `dml_dry_run` (the one+// Class III advisor) can migrate. See plans/2026-04-23-omni-tidb-completion-+// plan.md §1.5.0 invariant #4 + Bridge persistence rule.+//+// Mirrors backend/plugin/parser/mysql/omni.go's AsANTLRAST pattern:+// - Lazy parse + cache via pingcapParsed flag (matches mysql's antlrParsed).+// - Single bridge call per OmniAST instance regardless of how many advisors+// consume it (50+ per review under flip-last + cache).type OmniAST struct {// Node is the omni AST node (e.g. *ast.SelectStmt, *ast.CreateTableStmt).Node ast.Node@@ -33,6 +36,15 @@ type OmniAST struct {Text string// StartPosition is the 1-based position where this statement starts.StartPosition *storepb.Position++ // pingcapAST is lazily populated when AsPingCapAST() is called for the+ // first time. Cached for the lifetime of this OmniAST instance.+ // Will be removed once dml_dry_run migrates and the bridge is no longer+ // needed (post-Phase-2 §Tier 4g).+ pingcapAST *AST+ // pingcapParsed tracks whether we've attempted the native parse, to+ // distinguish "not yet parsed" from "parsed and got nil".+ pingcapParsed bool}// ASTStartPosition implements base.AST.@@ -40,6 +52,81 @@ func (a *OmniAST) ASTStartPosition() *storepb.Position {return a.StartPosition}+// AsPingCapAST implements PingCapASTProvider for backward compatibility with+// un-migrated advisors during the Phase 1.5 migration window. Lazily parses+// the SQL text with the native pingcap parser and caches the result.+//+// Implementation notes:+// - Calls ParseTiDB strictly (no error-recovery mode). Unlike mysql's+// parseSingleStatementLenient, pingcap has no equivalent error-recovery+// mode — it either parses fully or errors. For the bridge case (omni+// already validated the SQL syntax) strict parsing is correct.+// - Sets OriginTextPosition + per-column lines (CREATE TABLE) via the+// shared applyTiDBLineTracking helper, so post-flip un-migrated advisors+// that read node.OriginTextPosition() see the same line numbers as the+// pre-flip canonical path (ParseTiDBForSyntaxCheck).+//+// Returns (nil, false) if the native parser fails to parse the text. In that+// case un-migrated advisors get no AST for this statement and emit no advice+// — symmetric with the omni-side soft-fail in+// advisor/tidb/utils.go.getTiDBOmniNodes. This bridge protects review+// continuity when omni successfully wraps a statement but pingcap+// subsequently rejects it. The orthogonal failure mode — omni rejects at the+// dispatcher level so no *OmniAST exists for the bridge to operate on — is+// handled by the dispatcher-fallback contract (plan §1.5.0 invariant #8,+// shipping with the dispatcher flip in §1.5.N+1).+func (a *OmniAST) AsPingCapAST() (*AST, bool) {+ if a.pingcapParsed {+ return a.pingcapAST, a.pingcapAST != nil+ }+ a.pingcapParsed = true++ // Use the public ParseTiDB entry point — it configures the parser+ // consistently (window functions enabled, zero-date modes relaxed) so+ // the bridge result matches what direct ParseTiDB callers would see.+ nodes, err := ParseTiDB(a.Text, "", "")+ if err != nil {+ slog.Debug("pingcap re-parse failed in tidb omni bridge; un-migrated advisors will see no AST for this statement",+ slog.String("error", err.Error()),+ )+ return nil, false+ }+ if len(nodes) == 0 {+ slog.Debug("pingcap returned no nodes in tidb omni bridge; un-migrated advisors will see no AST for this statement")+ return nil, false+ }+ if len(nodes) > 1 {+ // Upstream contract: OmniAST.Text is always a single statement (set+ // by the dispatcher's split). nodes[1:] indicates a contract+ // violation upstream. Proceed with nodes[0] but record the surplus.+ slog.Debug("tidb omni bridge unexpectedly received multi-statement input; using nodes[0], dropping the rest",+ slog.Int("count", len(nodes)),+ )+ }+ node := nodes[0]++ // Mirror the line-tracking work that ParseTiDBForSyntaxCheck applies on+ // the canonical pre-flip path. Without this, post-flip un-migrated+ // advisors that call node.OriginTextPosition() would silently see line 1+ // instead of the statement's actual line in the multi-statement input.+ baseLine := 0+ if a.StartPosition != nil {+ baseLine = int(a.StartPosition.Line) - 1+ }+ if _, err := applyTiDBLineTracking(node, baseLine, a.Text); err != nil {+ slog.Debug("tidb omni bridge line-tracking failed; un-migrated advisors will see fallback line numbers",+ slog.String("error", err.Error()),+ )+ return nil, false+ }++ a.pingcapAST = &AST{+ StartPosition: a.StartPosition,+ Node: node,+ }+ return a.pingcapAST, true+}+// ParseTiDBOmni parses SQL using omni/tidb's parser and returns an ast.List.// This is the recommended entry point for new code that needs omni/tidb AST// nodes. Callers that need a pre-split []base.ParsedStatement should use thediff --git a/backend/plugin/parser/tidb/omni_test.go b/backend/plugin/parser/tidb/omni_test.goindex 70b47d6e89..a74a13f12d 100644--- a/backend/plugin/parser/tidb/omni_test.go+++ b/backend/plugin/parser/tidb/omni_test.go@@ -6,6 +6,7 @@ import ("github.com/stretchr/testify/require"storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base")func TestByteOffsetToRunePosition(t *testing.T) {@@ -144,3 +145,138 @@ func TestOmniASTStartPosition(t *testing.T) {o := &OmniAST{StartPosition: pos}a.Equal(pos, o.ASTStartPosition())}++// TestAsPingCapASTReturnsNativeAST pins the bridge contract: an OmniAST+// successfully parsed from omni-supported SQL also exposes a native pingcap+// AST via AsPingCapAST. Used by un-migrated advisors during the Phase 1.5+// migration window via tidbparser.GetTiDBAST.+func TestAsPingCapASTReturnsNativeAST(t *testing.T) {+ a := require.New(t)+ pos := &storepb.Position{Line: 1, Column: 1}+ o := &OmniAST{+ Text: "SELECT 1",+ StartPosition: pos,+ }++ got, ok := o.AsPingCapAST()+ a.True(ok, "expected pingcap fallback to succeed for valid SQL")+ a.NotNil(got)+ a.Equal(pos, got.StartPosition)+ a.NotNil(got.Node, "pingcap StmtNode should be populated")+}++// TestAsPingCapASTCachesAcrossCalls pins the lazy+cached contract: the+// native parse runs once per OmniAST instance regardless of how many+// advisors call the bridge. Mirrors mysql/OmniAST.AsANTLRAST's antlrParsed+// flag pattern.+func TestAsPingCapASTCachesAcrossCalls(t *testing.T) {+ a := require.New(t)+ o := &OmniAST{+ Text: "SELECT 1",+ StartPosition: &storepb.Position{Line: 1, Column: 1},+ }++ first, ok := o.AsPingCapAST()+ a.True(ok)+ a.NotNil(first)++ second, ok := o.AsPingCapAST()+ a.True(ok)++ // Same *AST pointer proves the cache hit; a fresh re-parse would+ // allocate a new wrapper.+ a.Same(first, second, "expected cached *AST; got fresh re-parse — bridge cache contract broken")+}++// TestAsPingCapASTCachesNegativeResult ensures a parse failure is also+// cached: a second call does not retry and continues to return (nil, false).+// Important so that an OmniAST wrapping unparseable-by-pingcap SQL doesn't+// repeatedly re-parse on every advisor call.+func TestAsPingCapASTCachesNegativeResult(t *testing.T) {+ a := require.New(t)+ o := &OmniAST{+ // Syntactically invalid SQL — pingcap should reject.+ Text: "SELECT FROM WHERE;",+ StartPosition: &storepb.Position{Line: 1, Column: 1},+ }++ got, ok := o.AsPingCapAST()+ a.False(ok)+ a.Nil(got)+ a.True(o.pingcapParsed, "pingcapParsed flag should be set after first attempt")++ // Second call returns the same negative result without re-attempting.+ got2, ok2 := o.AsPingCapAST()+ a.False(ok2)+ a.Nil(got2)+}++// TestGetTiDBASTFallsBackToProvider pins the cross-cutting contract: when+// the dispatcher returns *OmniAST, un-migrated callers of GetTiDBAST get a+// native *AST through the PingCapASTProvider fallback. Without this, the+// dispatcher flip in §1.5.N+1 silently breaks every un-migrated advisor.+func TestGetTiDBASTFallsBackToProvider(t *testing.T) {+ a := require.New(t)+ o := &OmniAST{+ Text: "SELECT 1",+ StartPosition: &storepb.Position{Line: 1, Column: 1},+ }++ got, ok := GetTiDBAST(o)+ a.True(ok, "GetTiDBAST should fall back to AsPingCapAST when handed an OmniAST")+ a.NotNil(got)+ a.NotNil(got.Node)+}++// TestAsPingCapASTLineTrackingMatchesCanonical pins that bridge-produced+// nodes have the same node.OriginTextPosition() as the canonical pre-flip+// path (ParseTiDBForSyntaxCheck). 49 of 51 tidb advisors call+// OriginTextPosition() to report advice line numbers; if the bridge omits+// the line-tracking work that the canonical path performs, post-flip+// un-migrated advisors silently regress to line 1 for every statement.+//+// This is the regression that PR review caught — the original bridge in+// this PR called bare ParseTiDB without applying applyTiDBLineTracking, so+// nodes came back with default OriginTextPosition (line 1 of the snippet,+// not line N of the original multi-statement input).+func TestAsPingCapASTLineTrackingMatchesCanonical(t *testing.T) {+ a := require.New(t)++ // Two-statement input. We compare the SECOND statement's line tracking+ // because that's where any drift would show: line 1 (default, broken)+ // vs line 2 (correct).+ multi := "CREATE TABLE foo (id INT);\nALTER TABLE foo ADD COLUMN x INT NOT NULL;"++ // Canonical pre-flip path: ParseTiDBForSyntaxCheck splits + parses ++ // applies applyTiDBLineTracking via the shared helper.+ canonical, err := ParseTiDBForSyntaxCheck(multi)+ a.NoError(err)+ a.Len(canonical, 2)+ canonicalSecond, ok := canonical[1].(*AST)+ a.True(ok)++ // Simulate what the dispatcher does post-flip: split, then for each+ // non-empty split build an OmniAST. The bridge must produce a node+ // whose OriginTextPosition matches the canonical path's.+ splits, err := base.SplitMultiSQL(storepb.Engine_TIDB, multi)+ a.NoError(err)+ a.GreaterOrEqual(len(splits), 2)+ secondSplit := splits[1]++ o := &OmniAST{+ Text: secondSplit.Text,+ // StartPosition is 1-based line in the original SQL; SplitMultiSQL+ // gives BaseLine() as 0-based, so +1 to convert.+ StartPosition: &storepb.Position{Line: int32(secondSplit.BaseLine()) + 1},+ }+ bridged, ok := o.AsPingCapAST()+ a.True(ok)++ a.Equal(+ canonicalSecond.Node.OriginTextPosition(),+ bridged.Node.OriginTextPosition(),+ "bridge OriginTextPosition must match ParseTiDBForSyntaxCheck — post-flip un-migrated advisors expect identical line numbers",+ )+ a.Equal(2, bridged.Node.OriginTextPosition(),+ "sanity: ALTER TABLE on line 2 of the input should report OriginTextPosition=2")+}diff --git a/backend/plugin/parser/tidb/tidb.go b/backend/plugin/parser/tidb/tidb.goindex 0bcb64244b..fe7093d51d 100644--- a/backend/plugin/parser/tidb/tidb.go+++ b/backend/plugin/parser/tidb/tidb.go@@ -61,17 +61,9 @@ func ParseTiDBForSyntaxCheck(statement string) ([]base.AST, error) {// and AST position matches Statement position.// Trim only at display points (e.g., error messages) where needed.- // Calculate the start line. The native TiDB parser may strip leading newlines- // from its internal Text(), so we count how many were stripped.- nativeText := node.Text()- leadingNewlinesStripped := strings.Count(singleSQL.Text, "\n") - strings.Count(nativeText, "\n")- actualStartLine := singleSQL.BaseLine() + leadingNewlinesStripped + 1-- node.SetOriginTextPosition(actualStartLine)- if n, ok := node.(*ast.CreateTableStmt); ok {- if err := SetLineForMySQLCreateTableStmt(n); err != nil {- return nil, errors.Wrapf(err, "failed to set line for create table statement at line %d", actualStartLine)- }+ actualStartLine, err := applyTiDBLineTracking(node, singleSQL.BaseLine(), singleSQL.Text)+ if err != nil {+ return nil, err}results = append(results, &AST{StartPosition: &storepb.Position{Line: int32(actualStartLine)},@@ -82,6 +74,34 @@ func ParseTiDBForSyntaxCheck(statement string) ([]base.AST, error) {return results, nil}+// applyTiDBLineTracking sets OriginTextPosition on a freshly-parsed pingcap+// node and walks CREATE TABLE columns to set their per-column line numbers+// via SetLineForMySQLCreateTableStmt. Used by both ParseTiDBForSyntaxCheck+// (the canonical pre-flip parsing path) and OmniAST.AsPingCapAST (the+// post-flip bridge for un-migrated advisors), so consumers reading+// node.OriginTextPosition() see consistent values across both paths.+//+// baseLine is the 0-based line index of the statement's first line in the+// original (pre-split) SQL — i.e., base.Statement.BaseLine().+//+// Returns the 1-based actualStartLine (`baseLine + leadingNewlinesStripped ++// 1`) so callers can use it for *AST.StartPosition.+func applyTiDBLineTracking(node ast.StmtNode, baseLine int, originalText string) (int, error) {+ // The native TiDB parser may strip leading newlines from its internal+ // Text(); count how many to recover the absolute line.+ nativeText := node.Text()+ leadingNewlinesStripped := strings.Count(originalText, "\n") - strings.Count(nativeText, "\n")+ actualStartLine := baseLine + leadingNewlinesStripped + 1++ node.SetOriginTextPosition(actualStartLine)+ if n, ok := node.(*ast.CreateTableStmt); ok {+ if err := SetLineForMySQLCreateTableStmt(n); err != nil {+ return actualStartLine, errors.Wrapf(err, "failed to set line for create table statement at line %d", actualStartLine)+ }+ }+ return actualStartLine, nil+}+// parseTiDBStatements is the ParseStatementsFunc for TiDB.// Returns []ParsedStatement with both text and AST populated.func parseTiDBStatements(statement string) ([]base.ParsedStatement, error) {