Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser/mssql): migrate SplitSQL and statement_type to omni AST (#19978)
rebelice85f2df4234b0
diff --git a/backend/plugin/advisor/mssql/rule_table_disallow_dml.go b/backend/plugin/advisor/mssql/rule_table_disallow_dml.goindex f6f31b0a91..68ea1d4fd9 100644--- a/backend/plugin/advisor/mssql/rule_table_disallow_dml.go+++ b/backend/plugin/advisor/mssql/rule_table_disallow_dml.go@@ -56,23 +56,23 @@ func (r *tableDisallowDMLRule) OnStatement(node ast.Node) {switch n := node.(type) {case *ast.InsertStmt:- if n.Relation != nil {- tableName = normalizeTableRef(n.Relation, "", "")+ if ref, ok := n.Relation.(*ast.TableRef); ok {+ tableName = normalizeTableRef(ref, "", "")loc = n.Loc}case *ast.UpdateStmt:- if n.Relation != nil {- tableName = normalizeTableRef(n.Relation, "", "")+ if ref, ok := n.Relation.(*ast.TableRef); ok {+ tableName = normalizeTableRef(ref, "", "")loc = n.Loc}case *ast.DeleteStmt:- if n.Relation != nil {- tableName = normalizeTableRef(n.Relation, "", "")+ if ref, ok := n.Relation.(*ast.TableRef); ok {+ tableName = normalizeTableRef(ref, "", "")loc = n.Loc}case *ast.MergeStmt:- if n.Target != nil {- tableName = normalizeTableRef(n.Target, "", "")+ if ref, ok := n.Target.(*ast.TableRef); ok {+ tableName = normalizeTableRef(ref, "", "")loc = n.Loc}case *ast.SelectStmt:diff --git a/backend/plugin/parser/tsql/query_span_extractor.go b/backend/plugin/parser/tsql/query_span_extractor.goindex 17f7be24b9..6e74425fda 100644--- a/backend/plugin/parser/tsql/query_span_extractor.go+++ b/backend/plugin/parser/tsql/query_span_extractor.go@@ -62,6 +62,14 @@ func (q *querySpanExtractor) getQuerySpan(ctx context.Context, statement string)return nil, err}+ if len(antlrASTs) == 0 {+ return &base.QuerySpan{+ Type: base.Select,+ SourceColumns: make(base.SourceColumnSet),+ Results: []base.QuerySpanResult{},+ }, nil+ }+if len(antlrASTs) != 1 {return nil, errors.Errorf("expected exactly 1 statement, got %d", len(antlrASTs))}diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.goindex 696e936fb6..fbf614d025 100644--- a/backend/plugin/parser/tsql/split.go+++ b/backend/plugin/parser/tsql/split.go@@ -1,10 +1,11 @@package tsqlimport (- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/tsql"+ "strings"++ omnimssql "github.com/bytebase/omni/mssql"+ "github.com/bytebase/omni/mssql/ast"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/plugin/parser/tokenizer"@@ -14,287 +15,76 @@ func init() {base.RegisterSplitterFunc(storepb.Engine_MSSQL, SplitSQL)}-// SplitSQL splits the given SQL statement into multiple SQL statements.+// SplitSQL splits the given T-SQL script into statements.+//+// Strategy: use omni's mssql parser to obtain per-statement byte ranges, then+// emit one base.Statement per omni statement with strict byte-contiguity and+// position adjacency:+//+// 1. stmts[0].Range.Start == 0+// stmts[i].Range.Start == stmts[i-1].Range.End (byte contiguity)+// stmts[0].Start == {Line: 1, Column: 1}+// stmts[i].Start == stmts[i-1].End (position adjacency)+//+// 2. `GO` batch separators become base.Statement entries with Empty=true.+// The GO batch semantics live in the tsqlbatch.Batcher (which strips GO+// before SplitSQL is called on the execution path); this function only+// reports them so that span analysis paths (which consume raw user SQL+// without the Batcher) can filter them cleanly via FilterEmptyStatements.+//+// On parser failure, falls back to the semicolon-based tokenizer splitter.func SplitSQL(statement string) ([]base.Statement, error) {- r, err := splitByParser(statement)+ omniStmts, err := omnimssql.Parse(statement)if err != nil {- // Fall back to semi split.return splitBySemi(statement)}- return r, err+ return toBaseStatements(statement, omniStmts), nil}func splitBySemi(statement string) ([]base.Statement, error) {t := tokenizer.NewTokenizer(statement)- list, err := t.SplitStandardMultiSQL()- if err != nil {- return nil, err- }- return list, nil+ return t.SplitStandardMultiSQL()}-func splitByParser(statement string) ([]base.Statement, error) {- inputStream := antlr.NewInputStream(statement)- lexer := parser.NewTSqlLexer(inputStream)- stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)- p := parser.NewTSqlParser(stream)-- // Remove default error listener and add our own error listener.- lexer.RemoveErrorListeners()- lexerErrorListener := &base.ParseErrorListener{- Statement: statement,- }- lexer.AddErrorListener(lexerErrorListener)-- p.RemoveErrorListeners()- parserErrorListener := &base.ParseErrorListener{- Statement: statement,- }- p.AddErrorListener(parserErrorListener)-- p.BuildParseTrees = true-- tree := p.Tsql_file()-- if lexerErrorListener.Err != nil {- return nil, lexerErrorListener.Err- }-- if parserErrorListener.Err != nil {- return nil, parserErrorListener.Err- }-- var result []base.Statement- tokens := stream.GetAllTokens()- start := 0- byteOffset := 0-- if len(tree.AllBatch_without_go()) == 0 {- // Go statement only.- for _, goStmt := range tree.AllGo_statement() {- pos := goStmt.GetStop().GetTokenIndex()- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: false,- })- byteOffset += stmtByteLength- start = pos + 1- }- return result, nil+// toBaseStatements converts omni's statement list into []base.Statement while+// enforcing byte-contiguity and position adjacency. Any bytes that omni did+// not attribute to a statement (gaps) are absorbed into the following+// statement so the output remains contiguous.+func toBaseStatements(sql string, omniStmts []omnimssql.Statement) []base.Statement {+ if len(omniStmts) == 0 {+ return nil}-- // First batch_without_go.- b := tree.AllBatch_without_go()[0]- var r []base.Statement- r, start, byteOffset = splitBatchWithoutGo(b, tokens, stream, start, byteOffset, statement)- result = append(result, r...)-- goIdx := 0- if len(tree.AllBatch_without_go()) > 1 {- bs := tree.AllBatch_without_go()[1:]- for _, b := range bs {- // Find all go statement before the current batch_without_go.- var goStmts []parser.IGo_statementContext- for _, goStmt := range tree.AllGo_statement()[goIdx:] {- if goStmt.GetStop().GetTokenIndex() < b.GetStart().GetTokenIndex() {- goStmts = append(goStmts, goStmt)- goIdx++- continue- }- break- }-- for _, goStmt := range goStmts {- pos := goStmt.GetStop().GetTokenIndex()- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: false,- })- byteOffset += stmtByteLength- start = pos + 1- }-- r, start, byteOffset = splitBatchWithoutGo(b, tokens, stream, start, byteOffset, statement)- result = append(result, r...)+ result := make([]base.Statement, 0, len(omniStmts))+ prevEnd := &storepb.Position{Line: 1, Column: 1}+ prevByte := 0+ for _, os := range omniStmts {+ startByte := prevByte+ endByte := os.ByteEnd+ if endByte < startByte {+ endByte = startByte}- }-- if goIdx < len(tree.AllGo_statement()) {- // Last go statement.- for _, goStmt := range tree.AllGo_statement()[goIdx:] {- pos := goStmt.GetStop().GetTokenIndex()- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: false,- })- byteOffset += stmtByteLength- start = pos + 1+ text := sql[startByte:endByte]+ endLine, endCol := base.CalculateLineAndColumn(sql, endByte)+ endPos := &storepb.Position{+ Line: int32(endLine + 1),+ Column: int32(endCol + 1),}- }- return result, nil-}+ _, isGo := os.AST.(*ast.GoStmt)+ empty := isGo || os.Empty() || strings.TrimSpace(text) == ""-func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token, stream *antlr.CommonTokenStream, start int, byteOffset int, statement string) ([]base.Statement, int, int) {- var result []base.Statement- switch {- case b.Batch_level_statement() == nil && b.Execute_body_batch() == nil:- // All sql_clauses.- for _, sqlClause := range b.AllSql_clauses() {- pos := sqlClause.GetStop().GetTokenIndex()- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: false,- })- byteOffset += stmtByteLength- start = pos + 1- }- case b.Batch_level_statement() != nil:- pos := b.Batch_level_statement().GetStop().GetTokenIndex()- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: false,- })- byteOffset += stmtByteLength- start = pos + 1- case b.Execute_body_batch() != nil:- pos := b.Execute_body_batch().GetStop().GetTokenIndex()- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{- Text: stmtText,+ Text: text,Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),+ Start: int32(startByte),+ End: int32(endByte),},- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: false,+ Start: prevEnd,+ End: endPos,+ Empty: empty,})- byteOffset += stmtByteLength- start = pos + 1- for _, sqlClause := range b.AllSql_clauses() {- pos := sqlClause.GetStop().GetTokenIndex()- stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])- stmtByteLength := len(stmtText)- // Calculate start position from byte offset (first character of Text)- startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)- result = append(result, base.Statement{- Text: stmtText,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(tokens[pos].GetLine()),- int32(tokens[pos].GetColumn()),- tokens[pos].GetText(),- ),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- Empty: false,- })- byteOffset += stmtByteLength- start = pos + 1- }- default:- // No statements found in this batch+ prevEnd = endPos+ prevByte = endByte}- return result, start, byteOffset+ return result}diff --git a/backend/plugin/parser/tsql/statement_type.go b/backend/plugin/parser/tsql/statement_type.goindex 6a5377bdb3..e700bbd0d5 100644--- a/backend/plugin/parser/tsql/statement_type.go+++ b/backend/plugin/parser/tsql/statement_type.go@@ -1,8 +1,7 @@package tsqlimport (- "github.com/antlr4-go/antlr/v4"- parser "github.com/bytebase/parser/tsql"+ "github.com/bytebase/omni/mssql/ast""github.com/pkg/errors"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -12,20 +11,12 @@ import (func GetStatementTypes(asts []base.AST) ([]storepb.StatementType, error) {sqlTypeSet := make(map[storepb.StatementType]bool)- for _, ast := range asts {- antlrAST, ok := base.GetANTLRAST(ast)+ for _, a := range asts {+ node, ok := GetOmniNode(a)if !ok {- return nil, errors.New("expected ANTLR AST for MSSQL")- }- for _, child := range antlrAST.Tree.GetChildren() {- _, ok := child.(*antlr.TerminalNodeImpl)- if ok {- continue- }-- t := getStatementType(child)- sqlTypeSet[t] = true+ return nil, errors.New("expected omni AST for MSSQL")}+ sqlTypeSet[getStatementType(node)] = true}var sqlTypes []storepb.StatementTypefor sqlType := range sqlTypeSet {@@ -34,59 +25,45 @@ func GetStatementTypes(asts []base.AST) ([]storepb.StatementType, error) {return sqlTypes, nil}-func getStatementType(node antlr.Tree) storepb.StatementType {- switch ctx := node.(type) {- case *parser.Tsql_fileContext, *parser.Batch_without_goContext, *parser.Batch_level_statementContext:- for _, child := range ctx.GetChildren() {- return getStatementType(child)- }- case *parser.Sql_clausesContext:- for _, child := range ctx.GetChildren() {- switch ctx := child.(type) {- case *parser.Ddl_clauseContext:- for _, child := range ctx.GetChildren() {- return getStatementType(child)- }- case *parser.Dml_clauseContext:- for _, child := range ctx.GetChildren() {- return getStatementType(child)- }- default:- }- }- case *parser.Alter_databaseContext:+func getStatementType(node ast.Node) storepb.StatementType {+ switch n := node.(type) {+ case *ast.AlterDatabaseStmt:return storepb.StatementType_ALTER_DATABASE- case *parser.Alter_indexContext:+ case *ast.AlterIndexStmt:return storepb.StatementType_ALTER_INDEX- case *parser.Alter_tableContext:+ case *ast.AlterTableStmt:return storepb.StatementType_ALTER_TABLE- case *parser.Create_databaseContext:+ case *ast.CreateDatabaseStmt:return storepb.StatementType_CREATE_DATABASE- case *parser.Create_indexContext:+ case *ast.CreateIndexStmt:return storepb.StatementType_CREATE_INDEX- case *parser.Create_schemaContext:+ case *ast.CreateSchemaStmt:return storepb.StatementType_CREATE_SCHEMA- case *parser.Create_tableContext:+ case *ast.CreateTableStmt:return storepb.StatementType_CREATE_TABLE- case *parser.Create_viewContext:+ case *ast.CreateViewStmt:return storepb.StatementType_CREATE_VIEW- case *parser.Drop_databaseContext:- return storepb.StatementType_DROP_DATABASE- case *parser.Drop_indexContext:- return storepb.StatementType_DROP_INDEX- case *parser.Drop_schemaContext:- return storepb.StatementType_DROP_SCHEMA- case *parser.Drop_tableContext:- return storepb.StatementType_DROP_TABLE- case *parser.Drop_viewContext:- return storepb.StatementType_DROP_VIEW- case *parser.Truncate_tableContext:+ case *ast.DropStmt:+ switch n.ObjectType {+ case ast.DropDatabase:+ return storepb.StatementType_DROP_DATABASE+ case ast.DropIndex:+ return storepb.StatementType_DROP_INDEX+ case ast.DropSchema:+ return storepb.StatementType_DROP_SCHEMA+ case ast.DropTable:+ return storepb.StatementType_DROP_TABLE+ case ast.DropView:+ return storepb.StatementType_DROP_VIEW+ default:+ }+ case *ast.TruncateStmt:return storepb.StatementType_TRUNCATE- case *parser.Delete_statementContext:+ case *ast.DeleteStmt:return storepb.StatementType_DELETE- case *parser.Insert_statementContext:+ case *ast.InsertStmt:return storepb.StatementType_INSERT- case *parser.Update_statementContext:+ case *ast.UpdateStmt:return storepb.StatementType_UPDATEdefault:}diff --git a/backend/plugin/parser/tsql/test-data/test_split.yaml b/backend/plugin/parser/tsql/test-data/test_split.yamlindex 8da8a5560e..7aca316000 100644--- a/backend/plugin/parser/tsql/test-data/test_split.yaml+++ b/backend/plugin/parser/tsql/test-data/test_split.yaml@@ -25,18 +25,30 @@start: 45end: 93empty: false- - text: "\n-- third statement\nselect * from @temp\n-- go statement\ngo"+ - text: "\n-- third statement\nselect * from @temp"baseline: 3start:line: 4column: 28+ end:+ line: 6+ column: 20+ range:+ start: 93+ end: 132+ empty: false+ - text: "\n-- go statement\ngo"+ baseline: 5+ start:+ line: 6+ column: 20end:line: 8column: 3range:- start: 93+ start: 132end: 151- empty: false+ empty: true- text: "\ngo"baseline: 7start:@@ -48,19 +60,31 @@range:start: 151end: 154- empty: false- - text: "\ngo\ngo"+ empty: true+ - text: "\ngo"baseline: 8start:line: 9column: 3end:- line: 11+ line: 10column: 3range:start: 154+ end: 157+ empty: true+ - text: "\ngo"+ baseline: 9+ start:+ line: 10+ column: 3+ end:+ line: 11+ column: 3+ range:+ start: 157end: 160- empty: false+ empty: true- description: multiple UPDATE statements with blank linesinput: "\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";\n\nUPDATE SalesLT.Address SET PostalCode = 0;\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";\n\n"result:diff --git a/go.mod b/go.modindex 2a2683cbe5..f9321b0f2e 100644--- a/go.mod+++ b/go.mod@@ -34,7 +34,7 @@ require (github.com/bmatcuk/doublestar/v4 v4.10.0github.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0- github.com/bytebase/omni v0.0.0-20260410092340-771f9d9350aa+ github.com/bytebase/omni v0.0.0-20260413035929-09d38c776926github.com/caarlos0/env/v11 v11.4.0github.com/cenkalti/backoff/v5 v5.0.3github.com/cockroachdb/cockroachdb-parser v0.25.2diff --git a/go.sum b/go.sumindex 74e4a4bf31..6fc9ecfe0f 100644--- a/go.sum+++ b/go.sum@@ -253,8 +253,8 @@ github.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888 h1:E8tz8maxpih/vtgithub.com/bytebase/gomongo v0.0.0-20260403033636-f6c50d4ab888/go.mod h1:IgLUOjyiHehdE0G3C92IjGYu9lJQgkPwf2mdNSvPWq8=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0 h1:bJ8NSf2K+pccl5m/wuu97fiYjDwpJH6IzURAq/3XA2U=github.com/bytebase/lsp-protocol v0.0.0-20250324071136-1586d0c10ff0/go.mod h1:BEWMfiZtqshdmulKJcIsWb45MUaEvDO4CROcgi8dfFg=-github.com/bytebase/omni v0.0.0-20260410092340-771f9d9350aa h1:P8rnhEtTELj6k9dO3TidjAYhFfnfSXL5Jcv1qDY3SoE=-github.com/bytebase/omni v0.0.0-20260410092340-771f9d9350aa/go.mod h1:AT+iLKCu7NyxTaYaOLzETRFBVurjRaqUU3YmFron9pw=+github.com/bytebase/omni v0.0.0-20260413035929-09d38c776926 h1:VeWXS00qYgw5ZsU0syf/Dor6VyqUjtvDqrAoHvT9p80=+github.com/bytebase/omni v0.0.0-20260413035929-09d38c776926/go.mod h1:AT+iLKCu7NyxTaYaOLzETRFBVurjRaqUU3YmFron9pw=github.com/bytebase/parser v0.0.0-20260324092042-1d52e5412f50 h1:Kq5wlhZ586Rch39y/IQV9MnrEzHob4iiXBG6zrbvSMU=github.com/bytebase/parser v0.0.0-20260324092042-1d52e5412f50/go.mod h1:jeak/EfutSOAuWKvrFIT2IZunhWprM7oTFBRgZ9RCxo=github.com/bytebase/pgx/v5 v5.0.0-20250212161523-96ff8aed8767 h1:YuR5G3LcxpO5ScSTA+Kdirxg9RW4TRc/v1t7N2Hfmbw=