Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(tidb): add omni parser shim (OmniAST, ParseTiDBOmni helpers) (#20107)
Vincent Huang1d0003edf296
diff --git a/backend/plugin/parser/tidb/omni.go b/backend/plugin/parser/tidb/omni.gonew file mode 100644index 0000000000..5ae031b5e8--- /dev/null+++ b/backend/plugin/parser/tidb/omni.go@@ -0,0 +1,103 @@+package tidb++import (+ "unicode/utf8"++ "github.com/bytebase/omni/tidb/ast"+ omniparser "github.com/bytebase/omni/tidb/parser"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++// OmniAST wraps an omni/tidb AST node and implements the base.AST interface.+//+// Unlike mysql/OmniAST, this type does NOT implement base.AntlrASTProvider+// (AsANTLRAST) nor a hypothetical native-pingcap fallback. Rationale:+// - TiDB has no TiDB-specific ANTLR grammar; tidb/backup.go uses the mysql+// ANTLR grammar on its own text and never consumes *OmniAST.+// - The 51 advisors under backend/plugin/advisor/tidb/ use getTiDBNodes() ->+// tidbparser.GetTiDBAST(stmt.AST) which asserts to the native *tidb.AST,+// not *OmniAST. So no legacy consumer sees *OmniAST today.+// - Task 1.2 (tidb.go extension) keeps the registered ParseStatementsFunc+// pointed at the native pingcap parser ("no behavioral change yet"), so+// *OmniAST won't reach advisors until a post-Phase 1 migration.+//+// When that migration flips the default and the 51 advisors need a compat+// path, add AsPingCapAST() + a matching provider interface in base/ast.go+// alongside the migration PR — with real consumers to exercise it.+type OmniAST struct {+ // Node is the omni AST node (e.g. *ast.SelectStmt, *ast.CreateTableStmt).+ Node ast.Node+ // Text is the original SQL text of this statement.+ Text string+ // StartPosition is the 1-based position where this statement starts.+ StartPosition *storepb.Position+}++// ASTStartPosition implements base.AST.+func (a *OmniAST) ASTStartPosition() *storepb.Position {+ return a.StartPosition+}++// ParseTiDBOmni parses SQL using omni/tidb's parser and returns an ast.List.+// This is the recommended entry point for new code that needs omni/tidb AST+// nodes. Callers that need a pre-split []base.ParsedStatement should use the+// registered base.ParseStatements with storepb.Engine_TIDB instead.+func ParseTiDBOmni(sql string) (*ast.List, error) {+ return omniparser.Parse(sql)+}++// GetOmniNode extracts the omni AST node from a base.AST interface.+// Returns the node and true if it is an *OmniAST, nil and false otherwise.+func GetOmniNode(a base.AST) (ast.Node, bool) {+ if a == nil {+ return nil, false+ }+ omniAST, ok := a.(*OmniAST)+ if !ok {+ return nil, false+ }+ return omniAST.Node, true+}++// ByteOffsetToRunePosition converts a byte offset in sql to a 1-based+// line:column position, where column is measured in Unicode code points+// (runes), matching storepb.Position semantics.+//+// byteOffset values past len(sql) are clamped to len(sql). Offsets that land+// inside a multi-byte UTF-8 sequence are treated as if they pointed at the+// start of the enclosing rune (they advance 0 runes past the last complete+// rune). Offsets before 0 are treated as 0.+func ByteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {+ if byteOffset < 0 {+ byteOffset = 0+ }+ if byteOffset > len(sql) {+ byteOffset = len(sql)+ }++ line := int32(1)+ runeCol := int32(0) // 0-based rune count on current line+ i := 0+ for i < byteOffset {+ r, size := utf8.DecodeRuneInString(sql[i:])+ if i+size > byteOffset {+ // Offset lands inside a multi-byte rune. Treat it as pointing at+ // the start of the enclosing rune — don't count it.+ break+ }+ if r == '\n' {+ line+++ runeCol = 0+ } else {+ runeCol+++ }+ i += size+ }++ return &storepb.Position{+ Line: line,+ Column: runeCol + 1, // convert to 1-based+ }+}diff --git a/backend/plugin/parser/tidb/omni_test.go b/backend/plugin/parser/tidb/omni_test.gonew file mode 100644index 0000000000..70b47d6e89--- /dev/null+++ b/backend/plugin/parser/tidb/omni_test.go@@ -0,0 +1,146 @@+package tidb++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++func TestByteOffsetToRunePosition(t *testing.T) {+ tests := []struct {+ name string+ sql string+ byteOffset int+ want *storepb.Position+ }{+ {+ name: "empty input, offset 0",+ sql: "",+ byteOffset: 0,+ want: &storepb.Position{Line: 1, Column: 1},+ },+ {+ name: "ascii start of line",+ sql: "SELECT 1",+ byteOffset: 0,+ want: &storepb.Position{Line: 1, Column: 1},+ },+ {+ name: "ascii mid-line",+ sql: "SELECT 1",+ byteOffset: 7,+ want: &storepb.Position{Line: 1, Column: 8},+ },+ {+ name: "ascii end of line",+ sql: "SELECT 1",+ byteOffset: 8,+ want: &storepb.Position{Line: 1, Column: 9},+ },+ {+ name: "newline advances line, resets column",+ sql: "SELECT 1\nSELECT 2",+ byteOffset: 9, // 'S' of second SELECT+ want: &storepb.Position{Line: 2, Column: 1},+ },+ {+ name: "multi-line, mid-line-2",+ sql: "SELECT 1\nSELECT 2",+ byteOffset: 16, // '2'+ want: &storepb.Position{Line: 2, Column: 8},+ },+ {+ name: "multi-byte BMP rune (Chinese ideograph, 3 bytes)",+ sql: "SELECT '中' AS x",+ byteOffset: 11, // byte AFTER the 3-byte ideograph — col should count it as 1 rune+ want: &storepb.Position{Line: 1, Column: 10},+ },+ {+ name: "surrogate-pair rune (emoji, 4 bytes) counted as ONE rune",+ sql: "SELECT '😀' x",+ byteOffset: 12, // byte AFTER the emoji — storepb column is rune-based so emoji = 1+ want: &storepb.Position{Line: 1, Column: 10},+ },+ {+ name: "mid-rune offset treated as start-of-rune (BMP 3-byte)",+ sql: "SELECT '中'",+ byteOffset: 9, // inside the 3-byte sequence (bytes 8,9,10 = E4 B8 AD)+ want: &storepb.Position{Line: 1, Column: 9},+ },+ {+ name: "mid-rune offset treated as start-of-rune (emoji)",+ sql: "a😀b",+ byteOffset: 3, // inside emoji (emoji occupies bytes 1..4)+ want: &storepb.Position{Line: 1, Column: 2},+ },+ {+ name: "offset clamped to len(sql) when past end",+ sql: "abc",+ byteOffset: 100,+ want: &storepb.Position{Line: 1, Column: 4},+ },+ {+ name: "negative offset treated as 0",+ sql: "abc",+ byteOffset: -5,+ want: &storepb.Position{Line: 1, Column: 1},+ },+ {+ name: "offset at the newline character is still on the pre-newline line",+ sql: "a\nb",+ byteOffset: 1, // the '\n' itself+ want: &storepb.Position{Line: 1, Column: 2},+ },+ {+ name: "multi-byte across a newline",+ sql: "中\n中",+ byteOffset: 4, // byte of second '中' = line 2, col 1+ want: &storepb.Position{Line: 2, Column: 1},+ },+ }++ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := ByteOffsetToRunePosition(tt.sql, tt.byteOffset)+ require.Equal(t, tt.want, got)+ })+ }+}++func TestGetOmniNode(t *testing.T) {+ a := require.New(t)++ // nil base.AST returns (nil, false).+ node, ok := GetOmniNode(nil)+ a.Nil(node)+ a.False(ok)++ // Non-OmniAST returns (nil, false). Use the native-tidb AST wrapper.+ nativeAST := &AST{StartPosition: &storepb.Position{Line: 1}}+ node, ok = GetOmniNode(nativeAST)+ a.Nil(node)+ a.False(ok)++ // OmniAST with non-nil node round-trips.+ list, err := ParseTiDBOmni("SELECT 1")+ a.NoError(err)+ a.NotNil(list)+ a.NotEmpty(list.Items)+ wrapped := &OmniAST{+ Node: list.Items[0],+ Text: "SELECT 1",+ StartPosition: &storepb.Position{Line: 1, Column: 1},+ }+ node, ok = GetOmniNode(wrapped)+ a.True(ok)+ a.Equal(list.Items[0], node)+}++func TestOmniASTStartPosition(t *testing.T) {+ a := require.New(t)+ pos := &storepb.Position{Line: 5, Column: 7}+ o := &OmniAST{StartPosition: pos}+ a.Equal(pos, o.ASTStartPosition())+}