Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(parser/tidb): flip dispatcher to omni-first with pingcap fallback (Phase 1.5 §1.5.N+1 — closes Phase 1.5) (#20340)

Vincent Huange99933725420
diff --git a/backend/plugin/advisor/tidb/advisor_builtin_prior_backup_check.go b/backend/plugin/advisor/tidb/advisor_builtin_prior_backup_check.go
index dc734f30d7..d149e875de 100644
--- a/backend/plugin/advisor/tidb/advisor_builtin_prior_backup_check.go
+++ b/backend/plugin/advisor/tidb/advisor_builtin_prior_backup_check.go
@@ -109,6 +109,15 @@ func (*StatementPriorBackupCheckAdvisor) Check(ctx context.Context, checkCtx adv
// Codex-fix-1c through 1e) demonstrated the brittle-enumeration
// pattern; switching DDL detection to pingcap's authoritative
// DDLNode interface eliminates the enumeration entirely.
+ //
+ // Post-flip cost note (Phase 1.5 §1.5.N+1): post-dispatcher-flip
+ // the dispatcher produces *OmniAST values, and getTiDBNodes routes
+ // each through OmniAST.AsPingCapAST() — a per-statement re-parse
+ // against pingcap. Lazy + cached per OmniAST instance (invariant
+ // #4), so the cost is bounded to one re-parse per statement-per-
+ // review for this dual-path advisor + dml_dry_run. No fix needed;
+ // bridge cleanup happens in Phase 2 when dml_dry_run migrates and
+ // the dual-path collapses.
pingcapStmts, err := getTiDBNodes(checkCtx)
if err != nil {
return nil, err
diff --git a/backend/plugin/advisor/tidb/utils.go b/backend/plugin/advisor/tidb/utils.go
index c47af1d49d..ce2438d4fc 100644
--- a/backend/plugin/advisor/tidb/utils.go
+++ b/backend/plugin/advisor/tidb/utils.go
@@ -708,11 +708,27 @@ const omniStmtsCacheKey = "tidb.omniStmts"
// getTiDBOmniNodes returns omni-parsed statements for migrated advisors.
//
-// Two invariants:
-// - Single-parse-per-review: result is cached on checkCtx.Memo, so all
-// migrated advisors in one review share one parse pass.
-// - Soft-fail per statement: omni parse errors are logged at debug and
-// the statement is skipped; the review never breaks on grammar gaps.
+// Post-flip (Phase 1.5 §1.5.N+1): the dispatcher in
+// backend/plugin/parser/tidb/dispatcher.go has already done the omni parse
+// and populated stmt.AST. This helper just unwraps OmniAST nodes and
+// preserves the cache contract.
+//
+// Three-arm type switch (per plan §1.5.0 invariant #8):
+// - *tidbparser.OmniAST: omni accepted — collect the node.
+// - *tidbparser.AST: dispatcher fell back to pingcap (omni rejected this
+// statement). Skip — migrated advisors emit no advice for omni-rejected
+// SQL. Soft-fail invariant preserved end-to-end (responsibility moved
+// from this helper to the dispatcher).
+// - default: unknown AST type. Warn (mandatory per invariant #8 — a
+// future engine introducing a third AST type for tidb shouldn't
+// silently drop statements).
+//
+// Cache contract (single-parse-per-review): the result slice is memoized
+// on checkCtx.Memo so subsequent calls within the same review return the
+// identical slice without re-walking the type switch. The dispatcher's
+// parse already happened once at split time, so this cache amortizes only
+// the type-switch + slice construction now — but the contract (one
+// observable parse per review) is preserved.
func getTiDBOmniNodes(checkCtx advisor.Context) ([]OmniStmt, error) {
if cached, ok := checkCtx.Memo(omniStmtsCacheKey); ok {
if stmts, typeOK := cached.([]OmniStmt); typeOK {
@@ -726,25 +742,25 @@ func getTiDBOmniNodes(checkCtx advisor.Context) ([]OmniStmt, error) {
var result []OmniStmt
for _, stmt := range checkCtx.ParsedStatements {
- if stmt.Empty {
- continue
- }
- list, err := tidbparser.ParseTiDBOmni(stmt.Text)
- if err != nil {
- slog.Debug("omni/tidb parse failed; skipping statement for omni-aware advisors",
- slog.String("error", err.Error()),
- )
+ if stmt.Empty || stmt.AST == nil {
continue
}
- if list == nil {
- continue
- }
- for _, item := range list.Items {
+ switch a := stmt.AST.(type) {
+ case *tidbparser.OmniAST:
result = append(result, OmniStmt{
- Node: item,
- Text: stmt.Text,
+ Node: a.Node,
+ Text: a.Text,
BaseLine: stmt.BaseLine(),
})
+ case *tidbparser.AST:
+ // Dispatcher fell back to pingcap for this statement (omni
+ // rejected). Migrated advisors skip it; un-migrated advisors
+ // using getTiDBNodes still see the pingcap AST. Soft-fail
+ // per invariant #2 preserved.
+ continue
+ default:
+ slog.Warn("unexpected stmt.AST type for tidb dispatcher",
+ slog.String("type", fmt.Sprintf("%T", a)))
}
}
diff --git a/backend/plugin/advisor/tidb/utils_test.go b/backend/plugin/advisor/tidb/utils_test.go
index d85d83945e..9a7c4edc57 100644
--- a/backend/plugin/advisor/tidb/utils_test.go
+++ b/backend/plugin/advisor/tidb/utils_test.go
@@ -15,11 +15,16 @@ import (
)
// TestGetTiDBOmniNodesCachesAcrossRules pins the Phase 1.5 single-parse-per-
-// review invariant: when multiple migrated advisors call getTiDBOmniNodes
-// with the same advisor.Context, omni parsing happens once and the result
-// slice is reused. Without this, parse cost scales with migrated-advisor
-// count and review latency degrades monotonically across the migration
-// window. See plans/2026-04-23-omni-tidb-completion-plan.md §1.5.0.
+// review invariant at the post-flip layer: when multiple migrated advisors
+// call getTiDBOmniNodes with the same advisor.Context, the type-switch +
+// slice construction happens once and the result slice is reused. Without
+// this, every migrated advisor pays the type-switch + allocation cost.
+//
+// Post-flip (§1.5.N+1) the dispatcher already parsed at split time, so
+// stmt.AST is populated with *OmniAST values directly. This test feeds
+// pre-populated ParsedStatements to mirror the dispatcher's output.
+//
+// See plans/2026-04-23-omni-tidb-completion-plan.md §1.5.0 invariant #1.
func TestGetTiDBOmniNodesCachesAcrossRules(t *testing.T) {
ctx := advisor.Context{
ParsedStatements: []base.ParsedStatement{
@@ -28,6 +33,10 @@ func TestGetTiDBOmniNodesCachesAcrossRules(t *testing.T) {
Text: "CREATE TABLE t (id INT)",
Start: &storepb.Position{Line: 1},
},
+ AST: &tidbparser.OmniAST{
+ Text: "CREATE TABLE t (id INT)",
+ StartPosition: &storepb.Position{Line: 1},
+ },
},
},
}
@@ -43,21 +52,28 @@ func TestGetTiDBOmniNodesCachesAcrossRules(t *testing.T) {
// Cache hit: same backing array returned both calls. Address equality
// on the first element proves the slice header was reused, not a fresh
- // re-parse that would allocate a new []OmniStmt.
+ // type-switch walk that would allocate a new []OmniStmt.
require.True(t, &first[0] == &second[0],
- "expected cached []OmniStmt; got fresh re-parse — single-parse-per-review invariant broken")
+ "expected cached []OmniStmt; got fresh walk — single-parse-per-review invariant broken")
}
-// TestGetTiDBOmniNodesSoftFailsOnGrammarGap pins the Phase 1.5 soft-fail
-// invariant: a statement that fails to parse with omni is logged and
-// skipped, not propagated as an error that breaks the advisor. The review
-// continues with whatever statements omni did parse.
+// TestGetTiDBOmniNodesSkipsPingcapFallbackAST pins the Phase 1.5 soft-fail
+// invariant at the post-flip layer: when the dispatcher fell back to
+// pingcap for a statement (omni rejected it), getTiDBOmniNodes silently
+// skips the *AST arm — migrated advisors emit no advice for omni-rejected
+// SQL, but un-migrated advisors using getTiDBNodes still see the pingcap
+// AST.
+//
+// Pre-flip this responsibility lived in getTiDBOmniNodes itself (which
+// re-parsed each statement and soft-failed on omni grammar gaps). Post-
+// flip §1.5.N+1, the responsibility moves to the dispatcher: the soft-
+// fail signal arrives as an *AST in stmt.AST instead of a parse error
+// inside getTiDBOmniNodes. The end-to-end soft-fail behavior is
+// preserved; only the layer changes.
//
-// This test feeds a statement that is valid TiDB SQL but uses a deferred
-// Phase 2 grammar feature (BATCH ... DRY RUN ...) that omni/tidb does not
-// yet support. The expectation is that getTiDBOmniNodes returns the
-// successfully-parsed statements and skips the BATCH one, with no error.
-func TestGetTiDBOmniNodesSoftFailsOnGrammarGap(t *testing.T) {
+// See plans/2026-04-23-omni-tidb-completion-plan.md §1.5.0 invariant #2
+// (soft-fail) + #8 (3-arm switch contract).
+func TestGetTiDBOmniNodesSkipsPingcapFallbackAST(t *testing.T) {
ctx := advisor.Context{
ParsedStatements: []base.ParsedStatement{
{
@@ -65,27 +81,41 @@ func TestGetTiDBOmniNodesSoftFailsOnGrammarGap(t *testing.T) {
Text: "CREATE TABLE t (id INT)",
Start: &storepb.Position{Line: 1},
},
+ AST: &tidbparser.OmniAST{
+ Text: "CREATE TABLE t (id INT)",
+ StartPosition: &storepb.Position{Line: 1},
+ },
},
{
Statement: base.Statement{
- Text: "BATCH ON id LIMIT 5000 UPDATE t SET v = v + 1",
+ Text: "FLASHBACK TABLE foo TO BEFORE DROP",
Start: &storepb.Position{Line: 2},
},
+ // Omni rejected this statement; dispatcher fell back to
+ // pingcap and produced an *AST. Migrated advisors must
+ // SKIP this statement silently — no advice, no error.
+ AST: &tidbparser.AST{
+ StartPosition: &storepb.Position{Line: 2},
+ },
},
{
Statement: base.Statement{
Text: "INSERT INTO t (id) VALUES (1)",
Start: &storepb.Position{Line: 3},
},
+ AST: &tidbparser.OmniAST{
+ Text: "INSERT INTO t (id) VALUES (1)",
+ StartPosition: &storepb.Position{Line: 3},
+ },
},
},
}
ctx.InitMemo()
got, err := getTiDBOmniNodes(ctx)
- require.NoError(t, err, "soft-fail invariant: parse errors must not propagate")
- require.GreaterOrEqual(t, len(got), 2,
- "expected at least the two non-BATCH statements to parse and be returned")
+ require.NoError(t, err, "soft-fail invariant: pingcap-fallback AST must not propagate as an error")
+ require.Len(t, got, 2,
+ "expected only the two omni-accepted statements; the *AST arm must be skipped")
}
// TestGetTiDBNodesSoftFailsOnBridgeMiss pins the post-flip soft-fail
@@ -209,6 +239,17 @@ func TestRunNamingConventionRuleEmitsInternalErrorAdvice(t *testing.T) {
Text: "CREATE INDEX idx_x ON t (a)",
Start: &storepb.Position{Line: 1},
},
+ // Post-flip §1.5.N+1: the dispatcher pre-populates AST.
+ // Synthetic *OmniAST wrapping a CreateIndexStmt so the
+ // collector closure proceeds to regex compilation.
+ AST: &tidbparser.OmniAST{
+ Node: &omniast.CreateIndexStmt{
+ IndexName: "idx_x",
+ Table: &omniast.TableRef{Name: "t"},
+ },
+ Text: "CREATE INDEX idx_x ON t (a)",
+ StartPosition: &storepb.Position{Line: 1},
+ },
},
},
}
diff --git a/backend/plugin/parser/tidb/dispatcher.go b/backend/plugin/parser/tidb/dispatcher.go
new file mode 100644
index 0000000000..2a107d1e54
--- /dev/null
+++ b/backend/plugin/parser/tidb/dispatcher.go
@@ -0,0 +1,195 @@
+package tidb
+
+import (
+ "errors"
+ "fmt"
+ "log/slog"
+
+ omniparser "github.com/bytebase/omni/tidb/parser"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// parseTiDBStatementsOmni is the post-flip ParseStatementsFunc for TiDB
+// (registered in tidb.go init). Implements Option B per plan §1.5.0
+// invariant #8: omni first, pingcap fallback per statement on omni parse
+// failure. The review never hard-fails at the dispatcher level on a Tier 4
+// grammar gap — customer sees advice for the parseable statements; the
+// omni-rejected ones get pingcap-AST so un-migrated advisors continue to
+// work, OR get no AST when both engines reject (the customer-facing error
+// then surfaces omni's complaint, matching the eventual Option A state).
+//
+// Per-fallback observability has TWO required surfaces (sub-contract):
+// - Counter tidb_dispatcher_omni_fallback_total{reason}: operations
+// signal that drives the eventual retirement decision. Debug logs are
+// dropped before reaching aggregation pipelines and cannot serve this
+// role.
+// - slog.Debug per fallback: developer signal for diagnosing individual
+// reports. Both surfaces ship together.
+//
+// Trade-off accepted: per-statement-skip-with-no-advice on omni-rejected
+// SQL (vs full-review-failure under Option A). Strictly better than Option
+// A for the migration window; less informative than the future state where
+// omni grammar is complete enough to drop the fallback.
+func parseTiDBStatementsOmni(statement string) ([]base.ParsedStatement, error) {
+ stmts, err := base.SplitMultiSQL(storepb.Engine_TIDB, statement)
+ if err != nil {
+ return nil, err
+ }
+
+ var result []base.ParsedStatement
+ for _, stmt := range stmts {
+ if stmt.Empty {
+ result = append(result, base.ParsedStatement{Statement: stmt})
+ continue
+ }
+
+ // Attempt order: omni first (sub-contract). Pingcap-first would
+ // defeat the architectural intent — post-flip, omni is canonical;
+ // pingcap is the safety net.
+ list, omniErr := ParseTiDBOmni(stmt.Text)
+ if omniErr == nil {
+ if list == nil || len(list.Items) == 0 {
+ // Omni succeeded but produced no items (e.g. comment-only
+ // statement that survived the splitter). Preserve the
+ // statement position with a nil AST, matching pre-flip
+ // parseTiDBStatements semantics.
+ result = append(result, base.ParsedStatement{Statement: stmt})
+ continue
+ }
+ for _, node := range list.Items {
+ result = append(result, base.ParsedStatement{
+ Statement: stmt,
+ AST: &OmniAST{
+ Node: node,
+ Text: stmt.Text,
+ StartPosition: stmt.Start,
+ },
+ })
+ }
+ continue
+ }
+
+ // Omni rejected — try Option B fallback to pingcap.
+ ast, fallbackErr := parsePingCapSingleStatement(stmt)
+ if fallbackErr != nil {
+ // Both engines reject. Don't increment the fallback counter —
+ // this is genuine bad SQL, not an omni grammar gap. Inflating
+ // the counter (especially the "unknown" bucket) on bad-SQL
+ // inputs would skew the Option B → A retirement-gate signal:
+ // after omni grammar is complete, malformed customer SQL
+ // would keep the counter non-zero and the gate would never
+ // fire. Surface omni's error so customer-facing expectations
+ // track the eventual Option A state (Q2 design choice — see
+ // plans/2026-04-23-omni-tidb-completion-plan.md §1.5.0
+ // invariant #8 + dispatcher_test.go regression pin).
+ return nil, convertOmniParseError(omniErr, stmt)
+ }
+
+ // Fallback succeeded — record the omni gap that pingcap bridged.
+ // Counter measures "omni rejected AND pingcap accepted" — the
+ // cases that genuinely justify Option B and drive the retirement
+ // decision.
+ reason := classifyOmniParseError(omniErr, stmt.Text)
+ tidbDispatcherOmniFallbackTotal.WithLabelValues(reason).Inc()
+ // Escalate "unknown" reason to Warn so ops sees the log line in
+ // production aggregation (which typically drops Debug). Without
+ // escalation the counter increments but the input excerpt + error
+ // string needed to add a new classifier pattern — and to drive
+ // the eventual Option B → A retirement gate to zero unknowns —
+ // stays invisible. Known reasons (flashback / sequence / batch_dml)
+ // stay at Debug: high-frequency, expected, counter-tracked.
+ logFn := slog.Debug
+ if reason == "unknown" {
+ logFn = slog.Warn
+ }
+ logFn("tidb dispatcher: omni parse failed; falling back to pingcap",
+ slog.String("reason", reason),
+ slog.String("excerpt", excerptForDebug(stmt.Text, 80)),
+ slog.String("error", omniErr.Error()),
+ )
+
+ ps := base.ParsedStatement{Statement: stmt}
+ if ast != nil {
+ ps.AST = ast
+ }
+ result = append(result, ps)
+ }
+
+ return result, nil
+}
+
+// parsePingCapSingleStatement runs the native pingcap parser on a single
+// pre-split base.Statement and applies line-tracking. Mirrors the per-loop
+// body of ParseTiDBForSyntaxCheck so the dispatcher's pingcap-fallback path
+// produces *AST values structurally identical to the canonical pre-flip
+// path (un-migrated advisors expect identical line numbers + node shape).
+//
+// Returns:
+// - (*AST, nil) on a clean single-node parse.
+// - (nil, nil) when the parser returned a non-1 node count (skip — same
+// semantic as ParseTiDBForSyntaxCheck's `if len(nodes) != 1 { continue }`).
+// - (nil, err) on a hard parser error, with the syntax error's line
+// adjusted to absolute coordinates.
+func parsePingCapSingleStatement(singleSQL base.Statement) (*AST, error) {
+ p := newTiDBParser()
+ nodes, _, err := p.Parse(singleSQL.Text, "", "")
+ if err != nil {
+ syntaxErr := convertParserError(err)
+ if se, ok := syntaxErr.(*base.SyntaxError); ok && se.Position != nil {
+ se.Position.Line = int32(singleSQL.BaseLine()) + se.Position.Line
+ }
+ return nil, syntaxErr
+ }
+ if len(nodes) != 1 {
+ return nil, nil
+ }
+ node := nodes[0]
+ actualStartLine, err := applyTiDBLineTracking(node, singleSQL.BaseLine(), singleSQL.Text)
+ if err != nil {
+ return nil, err
+ }
+ return &AST{
+ StartPosition: &storepb.Position{Line: int32(actualStartLine)},
+ Node: node,
+ }, nil
+}
+
+// convertOmniParseError mirrors mysql.go's convertOmniError: takes an omni
+// parser error and produces a base.SyntaxError with absolute line/column
+// coordinates (so the customer sees a position relative to the original
+// multi-statement input, not the per-statement excerpt).
+//
+// Returns the original error unchanged if it is not an *omniparser.ParseError.
+func convertOmniParseError(err error, stmt base.Statement) error {
+ var parseErr *omniparser.ParseError
+ if !errors.As(err, &parseErr) {
+ return err
+ }
+
+ pos := ByteOffsetToRunePosition(stmt.Text, parseErr.Position)
+ if stmt.Start != nil {
+ pos.Line += stmt.Start.Line - 1
+ }
+
+ msg := fmt.Sprintf("Syntax error at line %d:%d: %s", pos.Line, pos.Column, parseErr.Message)
+ if parseErr.RelatedText != "" {
+ msg += "\nrelated text: " + parseErr.RelatedText
+ }
+
+ return &base.SyntaxError{
+ Position: pos,
+ Message: msg,
+ RawMessage: parseErr.Message,
+ }
+}
+
+// excerptForDebug truncates s for slog.Debug payloads. Keeps fallback log
+// lines compact even when statement text runs long.
+func excerptForDebug(s string, maxRunes int) string {
+ if len(s) <= maxRunes {
+ return s
+ }
+ return s[:maxRunes] + "..."
+}
diff --git a/backend/plugin/parser/tidb/dispatcher_test.go b/backend/plugin/parser/tidb/dispatcher_test.go
new file mode 100644
index 0000000000..9421f37daf
--- /dev/null
+++ b/backend/plugin/parser/tidb/dispatcher_test.go
@@ -0,0 +1,169 @@
+package tidb
+
+import (
+ "strings"
+ "testing"
+
+ "github.com/prometheus/client_golang/prometheus/testutil"
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// TestParseTiDBStatementsOmni_OptionBFallback pins the Phase 1.5 §1.5.N+1
+// dispatcher flip contract (invariant #8): when omni rejects a statement,
+// the dispatcher falls back to native pingcap and the review never breaks.
+// Bracketed input asserts the omni-accepted statements come back as
+// *OmniAST and the omni-rejected one comes back as *AST (pingcap
+// fallback) — un-migrated advisors continue to work end-to-end.
+//
+// Also asserts the per-fallback observability sub-contract: the
+// tidb_dispatcher_omni_fallback_total{reason} counter increments by
+// exactly 1 for the BATCH statement, with the empirically-verified
+// "batch_dml" label.
+//
+// Empirical note: BATCH non-transactional DML is the canonical Option B
+// case — omni rejects (Tier 4 grammar gap; cumulative #30 dual-path
+// pattern), pingcap accepts. The plan's draft suggested
+// `FLASHBACK TABLE foo TO BEFORE DROP;` for this test, but pre-flip
+// empirical probe (invariant #9) found pingcap ALSO rejects that exact
+// syntax — it would test the both-engines-reject path, not Option B.
+// `BATCH ... DELETE` is the empirically-correct Option B input.
+func TestParseTiDBStatementsOmni_OptionBFallback(t *testing.T) {
+ const (
+ omniAccepted1 = "CREATE TABLE t (id INT);"
+ omniRejected = "BATCH ON id LIMIT 5000 DELETE FROM t WHERE 1=1;"
+ omniAccepted2 = "INSERT INTO t (id) VALUES (1);"
+ )
+ input := omniAccepted1 + "\n" + omniRejected + "\n" + omniAccepted2
+
+ // Snapshot the counter before so the assertion is delta-based — other
+ // tests in the package may have incremented it already.
+ before := testutil.ToFloat64(tidbDispatcherOmniFallbackTotal.WithLabelValues("batch_dml"))
+
+ result, err := parseTiDBStatementsOmni(input)
+ require.NoError(t, err,
+ "Option B contract: omni grammar gap on one statement must NOT propagate as a review-breaking error")
+ require.Len(t, result, 3,
+ "all three statements must be present in the result, even when omni rejects one")
+
+ // Assertion: accepted/rejected statements get the right AST type.
+ // The Statement.Text field carries the raw input for each split, so
+ // we identify by substring rather than index (split may include
+ // trailing whitespace etc.).
+ for _, ps := range result {
+ switch {
+ case strings.Contains(ps.Text, "BATCH"):
+ require.IsType(t, &AST{}, ps.AST,
+ "BATCH statement must carry pingcap *AST after Option B fallback (un-migrated advisors continue to function)")
+ case strings.Contains(ps.Text, "CREATE TABLE") || strings.Contains(ps.Text, "INSERT"):
+ require.IsType(t, &OmniAST{}, ps.AST,
+ "omni-accepted statements must carry *OmniAST")
+ default:
+ t.Fatalf("unexpected statement text in result: %q", ps.Text)
+ }
+ }
+
+ // Counter sub-contract.
+ after := testutil.ToFloat64(tidbDispatcherOmniFallbackTotal.WithLabelValues("batch_dml"))
+ require.InDelta(t, 1.0, after-before, 0.0001,
+ "counter must increment by exactly 1 for the single BATCH fallback")
+}
+
+// TestParseTiDBStatementsOmni_BothEnginesReject pins the Q2 design
+// choice: when omni AND pingcap both reject a statement, the dispatcher
+// surfaces omni's error, not pingcap's. This sets customer-facing
+// expectations matching the eventual Option A state — when the fallback
+// retires, the same input will still surface the same omni error.
+//
+// Also pins the "no counter inflation on both-reject" contract: malformed
+// SQL must NOT increment tidb_dispatcher_omni_fallback_total. Inflating
+// the counter (especially the "unknown" bucket) on bad-SQL inputs would
+// skew the Option B → A retirement-gate signal — after omni grammar is
+// complete, customer-side garbage SQL would keep the counter non-zero
+// and the gate would never fire. Per Codex round on PR #20340.
+func TestParseTiDBStatementsOmni_BothEnginesReject(t *testing.T) {
+ // SELECT FROM WHERE is genuine syntax garbage — both omni and pingcap
+ // reject it. Verified by the metrics_test parse-test (returns
+ // "unknown" classifier label).
+ const input = "SELECT FROM WHERE;"
+
+ beforeUnknown := testutil.ToFloat64(tidbDispatcherOmniFallbackTotal.WithLabelValues("unknown"))
+
+ _, err := parseTiDBStatementsOmni(input)
+ require.Error(t, err,
+ "both-engines-reject must propagate as an error to the customer")
+
+ // Surface should be omni's. Empirical omni error string for this
+ // input is `syntax error at or near "FROM" (line 1, column 8)`.
+ // After convertOmniParseError it is wrapped as a base.SyntaxError
+ // whose RawMessage carries omni's verbatim text.
+ syntaxErr, ok := err.(*base.SyntaxError)
+ require.True(t, ok, "error must be base.SyntaxError after conversion; got %T", err)
+ require.Contains(t, syntaxErr.RawMessage, "syntax error",
+ "raw message must come from omni's parser, preserving the eventual Option A surface")
+
+ // Counter contract: both-reject MUST NOT increment any reason bucket.
+ afterUnknown := testutil.ToFloat64(tidbDispatcherOmniFallbackTotal.WithLabelValues("unknown"))
+ require.Equal(t, beforeUnknown, afterUnknown,
+ "both-engines-reject must NOT inflate the fallback counter (retirement-gate signal stays clean)")
+}
+
+// TestParseTiDBStatementsOmni_AllAccepted pins the happy path: when omni
+// accepts every statement, no fallbacks fire and every result entry is
+// an *OmniAST. The counter does not move.
+func TestParseTiDBStatementsOmni_AllAccepted(t *testing.T) {
+ const input = "CREATE TABLE t (id INT); INSERT INTO t VALUES (1);"
+
+ beforeFlash := testutil.ToFloat64(tidbDispatcherOmniFallbackTotal.WithLabelValues("flashback"))
+ beforeUnknown := testutil.ToFloat64(tidbDispatcherOmniFallbackTotal.WithLabelValues("unknown"))
+
+ result, err := parseTiDBStatementsOmni(input)
+ require.NoError(t, err)
+ require.Len(t, result, 2)
+ for _, ps := range result {
+ require.IsType(t, &OmniAST{}, ps.AST,
+ "happy path: every statement must be *OmniAST")
+ }
+
+ require.Equal(t, beforeFlash,
+ testutil.ToFloat64(tidbDispatcherOmniFallbackTotal.WithLabelValues("flashback")),
+ "happy path: flashback counter must not move")
+ require.Equal(t, beforeUnknown,
+ testutil.ToFloat64(tidbDispatcherOmniFallbackTotal.WithLabelValues("unknown")),
+ "happy path: unknown counter must not move")
+}
+
+// TestParsePingCapSingleStatement_LineTrackingMatchesCanonical pins that
+// the dispatcher's pingcap fallback helper produces *AST values
+// structurally identical to the canonical pre-flip
+// ParseTiDBForSyntaxCheck path — so post-flip un-migrated advisors that
+// read node.OriginTextPosition() see consistent values.
+//
+// This is the dispatcher analog of the existing
+// TestAsPingCapASTLineTrackingMatchesCanonical (which pins the bridge
+// path); both fallback shapes (dispatcher fallback + bridge fallback)
+// must produce identical line numbers.
+func TestParsePingCapSingleStatement_LineTrackingMatchesCanonical(t *testing.T) {
+ // Multi-line input so the BaseLine offset matters.
+ const multi = "CREATE TABLE t1 (id INT);\n\nCREATE TABLE t2 (id INT);"
+
+ canonical, err := ParseTiDBForSyntaxCheck(multi)
+ require.NoError(t, err)
+ require.Len(t, canonical, 2)
+
+ stmts, err := base.SplitMultiSQL(storepb.Engine_TIDB, multi)
+ require.NoError(t, err)
+ require.Len(t, stmts, 2)
+
+ for i, stmt := range stmts {
+ got, err := parsePingCapSingleStatement(stmt)
+ require.NoError(t, err)
+ require.NotNil(t, got, "single-statement parse must succeed")
+ canonicalAST, ok := canonical[i].(*AST)
+ require.True(t, ok)
+ require.Equal(t, canonicalAST.Node.OriginTextPosition(), got.Node.OriginTextPosition(),
+ "dispatcher fallback line number must match canonical pre-flip path")
+ }
+}
diff --git a/backend/plugin/parser/tidb/metrics.go b/backend/plugin/parser/tidb/metrics.go
new file mode 100644
index 0000000000..f9da2981ad
--- /dev/null
+++ b/backend/plugin/parser/tidb/metrics.go
@@ -0,0 +1,74 @@
+package tidb
+
+import (
+ "strings"
+
+ "github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promauto"
+)
+
+// tidbDispatcherOmniFallbackTotal counts dispatcher fallback events per
+// classified reason. This is the operations signal that drives the eventual
+// Option B → A retirement decision (plan §1.5.0 invariant #8 + §Phase 3
+// "Telemetry instrumentation"): when fallback firing-frequency drops below a
+// threshold (e.g., < 0.1% of statements across customer reviews for 30 days),
+// the fallback can be removed and the dispatcher can hard-fail.
+//
+// Debug logs are NOT a substitute — production telemetry pipelines drop debug
+// logs before aggregation. Both surfaces (counter + slog.Debug per fallback)
+// ship together; missing either leaves Option B blind in different ways.
+//
+// Registered against prometheus.DefaultRegisterer. The /metrics endpoint at
+// backend/server/echo_routes.go folds DefaultGatherer with the echo-local
+// custom registry so this counter is scrape-visible.
+var tidbDispatcherOmniFallbackTotal = promauto.NewCounterVec(
+ prometheus.CounterOpts{
+ Name: "tidb_dispatcher_omni_fallback_total",
+ Help: "Count of TiDB dispatcher omni-parse failures that fell back to pingcap, labeled by classified reason.",
+ },
+ []string{"reason"},
+)
+
+// omniFallbackReasonPatterns maps known Tier-4-deferred grammar keywords to
+// counter labels. Each pattern is matched (case-insensitively) against the
+// concatenation of (omni error message) + " " + (input SQL).
+//
+// Why both sides of the haystack: empirically (probed against omni
+// v0.0.0-20260513072939-39c04c4cca0f) the omni parser echoes the offending
+// keyword for FLASHBACK and BATCH but reports CREATE SEQUENCE as
+// "unexpected token after CREATE" — the keyword SEQUENCE never appears in
+// the error string. Without input-side matching we'd silently lose the
+// SEQUENCE telemetry signal entirely.
+//
+// PRECEDENCE: first-match-wins iteration order. The current three Tier-4
+// patterns (FLASHBACK / SEQUENCE / BATCH) have no overlap, so order is
+// presently moot. If a future Tier-4 grammar gap introduces an overlapping
+// keyword, the patterns list will need explicit ordering or
+// more-specific-first sorting; this comment documents the contract before
+// it becomes load-bearing.
+var omniFallbackReasonPatterns = []struct {
+ pattern string // uppercase substring; matched against UPPER(err.Error() + " " + sql)
+ reason string // counter label value
+}{
+ {"FLASHBACK", "flashback"},
+ {"SEQUENCE", "sequence"},
+ {"BATCH", "batch_dml"},
+}
+
+// classifyOmniParseError maps an omni parse error + the input SQL into a
+// counter label. The input SQL is required because the omni error string
+// does not always echo the offending keyword (notably for CREATE SEQUENCE).
+// Returns "unknown" for nil error or any error+sql whose haystack contains
+// no known Tier-4 keyword.
+func classifyOmniParseError(err error, sql string) string {
+ if err == nil {
+ return "unknown"
+ }
+ haystack := strings.ToUpper(err.Error() + " " + sql)
+ for _, p := range omniFallbackReasonPatterns {
+ if strings.Contains(haystack, p.pattern) {
+ return p.reason
+ }
+ }
+ return "unknown"
+}
diff --git a/backend/plugin/parser/tidb/metrics_test.go b/backend/plugin/parser/tidb/metrics_test.go
new file mode 100644
index 0000000000..d0df5a3038
--- /dev/null
+++ b/backend/plugin/parser/tidb/metrics_test.go
@@ -0,0 +1,91 @@
+package tidb
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// TestClassifyOmniParseError pins the dispatcher fallback classifier
+// against the actual omni error strings — empirically derived against
+// omni v0.0.0-20260513072939-39c04c4cca0f. The classifier must label each
+// known Tier-4 grammar gap correctly so the
+// tidb_dispatcher_omni_fallback_total{reason} counter drives the
+// invariant #8 retirement gate (otherwise we ship Option B blind).
+//
+// Critical empirical finding (motivating the err+sql two-arg signature):
+// CREATE SEQUENCE seq; produces an omni error string of "unexpected token
+// after CREATE (line 1, column 8)" — the keyword SEQUENCE never appears
+// in the error. A classifier that matched only against err.Error() would
+// silently mis-label SEQUENCE rejections as "unknown" and lose the entire
+// SEQUENCE telemetry signal. The sql parameter is required, not optional.
+func TestClassifyOmniParseError(t *testing.T) {
+ cases := []struct {
+ name string
+ sql string
+ wantOK bool // expect parse to fail
+ expected string // counter label
+ }{
+ {
+ name: "FLASHBACK keyword in error msg AND input",
+ sql: "FLASHBACK TABLE foo TO BEFORE DROP;",
+ wantOK: false,
+ expected: "flashback",
+ },
+ {
+ name: "SEQUENCE keyword ONLY in input (omni err says 'after CREATE')",
+ sql: "CREATE SEQUENCE seq;",
+ wantOK: false,
+ expected: "sequence",
+ },
+ {
+ name: "BATCH keyword in error msg AND input",
+ sql: "BATCH ON id LIMIT 100 DELETE FROM t WHERE 1=1;",
+ wantOK: false,
+ expected: "batch_dml",
+ },
+ {
+ name: "lowercase batch in input is matched (case-insensitive)",
+ sql: "batch on id limit 100 delete from t where 1=1;",
+ wantOK: false,
+ expected: "batch_dml",
+ },
+ {
+ name: "genuine syntax error → unknown (no Tier-4 keyword present)",
+ sql: "SELECT FROM WHERE;",
+ wantOK: false,
+ expected: "unknown",
+ },
+ }
+
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := ParseTiDBOmni(tc.sql)
+ require.Error(t, err, "test setup expects this input to be rejected by current omni")
+ got := classifyOmniParseError(err, tc.sql)
+ require.Equal(t, tc.expected, got,
+ "classifier must return the correct label for omni err %q + sql %q",
+ err.Error(), tc.sql)
+ })
+ }
+}
+
+// TestClassifyOmniParseError_NilErr pins the contract that a nil error
+// returns "unknown" — the classifier is called only on the fallback path
+// in practice, but defensive against future refactors that might call it
+// on success.
+func TestClassifyOmniParseError_NilErr(t *testing.T) {
+ require.Equal(t, "unknown", classifyOmniParseError(nil, "anything"))
+}
+
+// TestClassifyOmniParseError_NonOmniErr pins behavior for an error type
+// that is not an *omniparser.ParseError — the haystack is just
+// err.Error(), so non-omni errors get classified normally too. Useful for
+// future-proofing if the dispatcher ever wraps the omni error.
+func TestClassifyOmniParseError_NonOmniErr(t *testing.T) {
+ plainErr := errors.New("some FLASHBACK-shaped wrapper")
+ require.Equal(t, "flashback",
+ classifyOmniParseError(plainErr, "irrelevant"),
+ "classifier matches the haystack regardless of error type")
+}
diff --git a/backend/plugin/parser/tidb/tidb.go b/backend/plugin/parser/tidb/tidb.go
index fe7093d51d..2ef430b5d9 100644
--- a/backend/plugin/parser/tidb/tidb.go
+++ b/backend/plugin/parser/tidb/tidb.go
@@ -20,55 +20,39 @@ import (
)
func init() {
- base.RegisterParseStatementsFunc(storepb.Engine_TIDB, parseTiDBStatements)
+ // Phase 1.5 §1.5.N+1 dispatcher flip: register the Option B
+ // omni-first/pingcap-fallback dispatcher (parseTiDBStatementsOmni in
+ // dispatcher.go). See plans/2026-04-23-omni-tidb-completion-plan.md
+ // §1.5.0 invariant #8 for the contract.
+ base.RegisterParseStatementsFunc(storepb.Engine_TIDB, parseTiDBStatementsOmni)
base.RegisterGetStatementTypes(storepb.Engine_TIDB, GetStatementTypes)
}
// ParseTiDBForSyntaxCheck parses TiDB SQL for syntax checking purposes.
-// Returns []base.AST with *TiDBAST instances.
+// Returns []base.AST with *AST instances.
+//
+// Per-statement parse + line-tracking is delegated to
+// parsePingCapSingleStatement (in dispatcher.go) so the dispatcher's
+// pingcap-fallback path and this canonical pre-flip path produce
+// structurally identical *AST values. (nil, nil) from the helper
+// signals "non-1 node count, skip" — same semantic as the pre-
+// refactor inline `if len(nodes) != 1 { continue }`.
func ParseTiDBForSyntaxCheck(statement string) ([]base.AST, error) {
singleSQLs, err := base.SplitMultiSQL(storepb.Engine_TIDB, statement)
if err != nil {
return nil, err
}
- p := newTiDBParser()
var results []base.AST
for _, singleSQL := range singleSQLs {
- nodes, _, err := p.Parse(singleSQL.Text, "", "")
+ ast, err := parsePingCapSingleStatement(singleSQL)
if err != nil {
- // Convert parser error to SyntaxError with proper position
- syntaxErr := convertParserError(err)
- // Adjust the line number to be absolute (relative to the full statement)
- // The TiDB parser reports line numbers relative to singleSQL.Text (starting at 1)
- // We need to add the offset to get the absolute line number
- if se, ok := syntaxErr.(*base.SyntaxError); ok && se.Position != nil {
- // errorLine is 1-based relative to singleSQL.Text
- // singleSQL.BaseLine() is 0-based line number of the first line in the original statement
- // Absolute line (1-based) = BaseLine (0-based) + errorLine (1-based)
- se.Position.Line = int32(singleSQL.BaseLine()) + se.Position.Line
- }
- return nil, syntaxErr
+ return nil, err
}
-
- if len(nodes) != 1 {
+ if ast == nil {
continue
}
-
- node := nodes[0]
- // node.Text() includes leading whitespace from singleSQL.Text.
- // This maintains consistency: Statement.Start points to first char of Statement.Text,
- // and AST position matches Statement position.
- // Trim only at display points (e.g., error messages) where needed.
-
- actualStartLine, err := applyTiDBLineTracking(node, singleSQL.BaseLine(), singleSQL.Text)
- if err != nil {
- return nil, err
- }
- results = append(results, &AST{
- StartPosition: &storepb.Position{Line: int32(actualStartLine)},
- Node: node,
- })
+ results = append(results, ast)
}
return results, nil
@@ -102,38 +86,6 @@ func applyTiDBLineTracking(node ast.StmtNode, baseLine int, originalText string)
return actualStartLine, nil
}
-// parseTiDBStatements is the ParseStatementsFunc for TiDB.
-// Returns []ParsedStatement with both text and AST populated.
-func parseTiDBStatements(statement string) ([]base.ParsedStatement, error) {
- // First split to get Statement with text and positions
- stmts, err := base.SplitMultiSQL(storepb.Engine_TIDB, statement)
- if err != nil {
- return nil, err
- }
-
- // Then parse to get ASTs
- asts, err := ParseTiDBForSyntaxCheck(statement)
- if err != nil {
- return nil, err
- }
-
- // Combine: Statement provides text/positions, AST provides parsed tree
- var result []base.ParsedStatement
- astIndex := 0
- for _, stmt := range stmts {
- ps := base.ParsedStatement{
- Statement: stmt,
- }
- if !stmt.Empty && astIndex < len(asts) {
- ps.AST = asts[astIndex]
- astIndex++
- }
- result = append(result, ps)
- }
-
- return result, nil
-}
-
func newTiDBParser() *tidbparser.Parser {
p := tidbparser.New()
diff --git a/backend/server/echo_routes.go b/backend/server/echo_routes.go
index 2807c648ec..9447534c57 100644
--- a/backend/server/echo_routes.go
+++ b/backend/server/echo_routes.go
@@ -10,6 +10,7 @@ import (
"github.com/labstack/echo/v5/middleware"
"github.com/pkg/errors"
"github.com/prometheus/client_golang/prometheus"
+ "github.com/prometheus/client_golang/prometheus/promhttp"
connectcors "connectrpc.com/cors"
@@ -68,9 +69,30 @@ func configureEchoRouters(
Subsystem: "api",
Registerer: registry,
}))
- e.GET("/metrics", echoprometheus.NewHandlerWithConfig(echoprometheus.HandlerConfig{
- Gatherer: registry,
- }))
+ // Fold the local echo registry with the default registry at scrape
+ // time. The local registry isolates echo HTTP middleware metrics from
+ // duplicate-registration errors in tests; the default registry catches
+ // promauto-registered metrics from other packages (e.g. db_metrics,
+ // the tidb dispatcher fallback counter, and Go runtime metrics auto-
+ // registered by client_golang). Without this fold, those metrics are
+ // registered but never exposed at /metrics.
+ //
+ // Why bypass echoprometheus.NewHandlerWithConfig: that helper only
+ // applies promhttp.InstrumentMetricHandler when its Gatherer also
+ // implements prometheus.Registerer (echoprometheus/prometheus.go:129).
+ // prometheus.Gatherers (slice type) does not implement Registerer,
+ // so passing the fold there silently drops scrape-health
+ // self-instrumentation (promhttp_metric_handler_requests_total etc.).
+ // Use promhttp directly: pass the local registry as the Registerer
+ // for self-instrumentation; pass the Gatherers fold as the gather
+ // source. Both observability surfaces preserved.
+ e.GET("/metrics", echo.WrapHandler(promhttp.InstrumentMetricHandler(
+ registry,
+ promhttp.HandlerFor(
+ prometheus.Gatherers{registry, prometheus.DefaultGatherer},
+ promhttp.HandlerOpts{},
+ ),
+ )))
e.GET("/healthz", func(c *echo.Context) error {
return c.String(http.StatusOK, "OK")