Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(tidb): add omni parser shim (OmniAST, ParseTiDBOmni helpers) (#20107)

Vincent Huang1d0003edf296
diff --git a/backend/plugin/parser/tidb/omni.go b/backend/plugin/parser/tidb/omni.go
new file mode 100644
index 0000000000..5ae031b5e8
--- /dev/null
+++ b/backend/plugin/parser/tidb/omni.go
@@ -0,0 +1,103 @@
+package tidb
+
+import (
+ "unicode/utf8"
+
+ "github.com/bytebase/omni/tidb/ast"
+ omniparser "github.com/bytebase/omni/tidb/parser"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+// OmniAST wraps an omni/tidb AST node and implements the base.AST interface.
+//
+// Unlike mysql/OmniAST, this type does NOT implement base.AntlrASTProvider
+// (AsANTLRAST) nor a hypothetical native-pingcap fallback. Rationale:
+// - TiDB has no TiDB-specific ANTLR grammar; tidb/backup.go uses the mysql
+// ANTLR grammar on its own text and never consumes *OmniAST.
+// - The 51 advisors under backend/plugin/advisor/tidb/ use getTiDBNodes() ->
+// tidbparser.GetTiDBAST(stmt.AST) which asserts to the native *tidb.AST,
+// not *OmniAST. So no legacy consumer sees *OmniAST today.
+// - Task 1.2 (tidb.go extension) keeps the registered ParseStatementsFunc
+// pointed at the native pingcap parser ("no behavioral change yet"), so
+// *OmniAST won't reach advisors until a post-Phase 1 migration.
+//
+// When that migration flips the default and the 51 advisors need a compat
+// path, add AsPingCapAST() + a matching provider interface in base/ast.go
+// alongside the migration PR — with real consumers to exercise it.
+type OmniAST struct {
+ // Node is the omni AST node (e.g. *ast.SelectStmt, *ast.CreateTableStmt).
+ Node ast.Node
+ // Text is the original SQL text of this statement.
+ Text string
+ // StartPosition is the 1-based position where this statement starts.
+ StartPosition *storepb.Position
+}
+
+// ASTStartPosition implements base.AST.
+func (a *OmniAST) ASTStartPosition() *storepb.Position {
+ return a.StartPosition
+}
+
+// ParseTiDBOmni parses SQL using omni/tidb's parser and returns an ast.List.
+// This is the recommended entry point for new code that needs omni/tidb AST
+// nodes. Callers that need a pre-split []base.ParsedStatement should use the
+// registered base.ParseStatements with storepb.Engine_TIDB instead.
+func ParseTiDBOmni(sql string) (*ast.List, error) {
+ return omniparser.Parse(sql)
+}
+
+// GetOmniNode extracts the omni AST node from a base.AST interface.
+// Returns the node and true if it is an *OmniAST, nil and false otherwise.
+func GetOmniNode(a base.AST) (ast.Node, bool) {
+ if a == nil {
+ return nil, false
+ }
+ omniAST, ok := a.(*OmniAST)
+ if !ok {
+ return nil, false
+ }
+ return omniAST.Node, true
+}
+
+// ByteOffsetToRunePosition converts a byte offset in sql to a 1-based
+// line:column position, where column is measured in Unicode code points
+// (runes), matching storepb.Position semantics.
+//
+// byteOffset values past len(sql) are clamped to len(sql). Offsets that land
+// inside a multi-byte UTF-8 sequence are treated as if they pointed at the
+// start of the enclosing rune (they advance 0 runes past the last complete
+// rune). Offsets before 0 are treated as 0.
+func ByteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {
+ if byteOffset < 0 {
+ byteOffset = 0
+ }
+ if byteOffset > len(sql) {
+ byteOffset = len(sql)
+ }
+
+ line := int32(1)
+ runeCol := int32(0) // 0-based rune count on current line
+ i := 0
+ for i < byteOffset {
+ r, size := utf8.DecodeRuneInString(sql[i:])
+ if i+size > byteOffset {
+ // Offset lands inside a multi-byte rune. Treat it as pointing at
+ // the start of the enclosing rune — don't count it.
+ break
+ }
+ if r == '\n' {
+ line++
+ runeCol = 0
+ } else {
+ runeCol++
+ }
+ i += size
+ }
+
+ return &storepb.Position{
+ Line: line,
+ Column: runeCol + 1, // convert to 1-based
+ }
+}
diff --git a/backend/plugin/parser/tidb/omni_test.go b/backend/plugin/parser/tidb/omni_test.go
new file mode 100644
index 0000000000..70b47d6e89
--- /dev/null
+++ b/backend/plugin/parser/tidb/omni_test.go
@@ -0,0 +1,146 @@
+package tidb
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+func TestByteOffsetToRunePosition(t *testing.T) {
+ tests := []struct {
+ name string
+ sql string
+ byteOffset int
+ want *storepb.Position
+ }{
+ {
+ name: "empty input, offset 0",
+ sql: "",
+ byteOffset: 0,
+ want: &storepb.Position{Line: 1, Column: 1},
+ },
+ {
+ name: "ascii start of line",
+ sql: "SELECT 1",
+ byteOffset: 0,
+ want: &storepb.Position{Line: 1, Column: 1},
+ },
+ {
+ name: "ascii mid-line",
+ sql: "SELECT 1",
+ byteOffset: 7,
+ want: &storepb.Position{Line: 1, Column: 8},
+ },
+ {
+ name: "ascii end of line",
+ sql: "SELECT 1",
+ byteOffset: 8,
+ want: &storepb.Position{Line: 1, Column: 9},
+ },
+ {
+ name: "newline advances line, resets column",
+ sql: "SELECT 1\nSELECT 2",
+ byteOffset: 9, // 'S' of second SELECT
+ want: &storepb.Position{Line: 2, Column: 1},
+ },
+ {
+ name: "multi-line, mid-line-2",
+ sql: "SELECT 1\nSELECT 2",
+ byteOffset: 16, // '2'
+ want: &storepb.Position{Line: 2, Column: 8},
+ },
+ {
+ name: "multi-byte BMP rune (Chinese ideograph, 3 bytes)",
+ sql: "SELECT '中' AS x",
+ byteOffset: 11, // byte AFTER the 3-byte ideograph — col should count it as 1 rune
+ want: &storepb.Position{Line: 1, Column: 10},
+ },
+ {
+ name: "surrogate-pair rune (emoji, 4 bytes) counted as ONE rune",
+ sql: "SELECT '😀' x",
+ byteOffset: 12, // byte AFTER the emoji — storepb column is rune-based so emoji = 1
+ want: &storepb.Position{Line: 1, Column: 10},
+ },
+ {
+ name: "mid-rune offset treated as start-of-rune (BMP 3-byte)",
+ sql: "SELECT '中'",
+ byteOffset: 9, // inside the 3-byte sequence (bytes 8,9,10 = E4 B8 AD)
+ want: &storepb.Position{Line: 1, Column: 9},
+ },
+ {
+ name: "mid-rune offset treated as start-of-rune (emoji)",
+ sql: "a😀b",
+ byteOffset: 3, // inside emoji (emoji occupies bytes 1..4)
+ want: &storepb.Position{Line: 1, Column: 2},
+ },
+ {
+ name: "offset clamped to len(sql) when past end",
+ sql: "abc",
+ byteOffset: 100,
+ want: &storepb.Position{Line: 1, Column: 4},
+ },
+ {
+ name: "negative offset treated as 0",
+ sql: "abc",
+ byteOffset: -5,
+ want: &storepb.Position{Line: 1, Column: 1},
+ },
+ {
+ name: "offset at the newline character is still on the pre-newline line",
+ sql: "a\nb",
+ byteOffset: 1, // the '\n' itself
+ want: &storepb.Position{Line: 1, Column: 2},
+ },
+ {
+ name: "multi-byte across a newline",
+ sql: "中\n中",
+ byteOffset: 4, // byte of second '中' = line 2, col 1
+ want: &storepb.Position{Line: 2, Column: 1},
+ },
+ }
+
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := ByteOffsetToRunePosition(tt.sql, tt.byteOffset)
+ require.Equal(t, tt.want, got)
+ })
+ }
+}
+
+func TestGetOmniNode(t *testing.T) {
+ a := require.New(t)
+
+ // nil base.AST returns (nil, false).
+ node, ok := GetOmniNode(nil)
+ a.Nil(node)
+ a.False(ok)
+
+ // Non-OmniAST returns (nil, false). Use the native-tidb AST wrapper.
+ nativeAST := &AST{StartPosition: &storepb.Position{Line: 1}}
+ node, ok = GetOmniNode(nativeAST)
+ a.Nil(node)
+ a.False(ok)
+
+ // OmniAST with non-nil node round-trips.
+ list, err := ParseTiDBOmni("SELECT 1")
+ a.NoError(err)
+ a.NotNil(list)
+ a.NotEmpty(list.Items)
+ wrapped := &OmniAST{
+ Node: list.Items[0],
+ Text: "SELECT 1",
+ StartPosition: &storepb.Position{Line: 1, Column: 1},
+ }
+ node, ok = GetOmniNode(wrapped)
+ a.True(ok)
+ a.Equal(list.Items[0], node)
+}
+
+func TestOmniASTStartPosition(t *testing.T) {
+ a := require.New(t)
+ pos := &storepb.Position{Line: 5, Column: 7}
+ o := &OmniAST{StartPosition: pos}
+ a.Equal(pos, o.ASTStartPosition())
+}