Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: make omni split position mapping linear (#20556)
rebelice7585603766af
diff --git a/backend/plugin/parser/base/position.go b/backend/plugin/parser/base/position.goindex e298e668cb..d9884409a6 100644--- a/backend/plugin/parser/base/position.go+++ b/backend/plugin/parser/base/position.go@@ -37,10 +37,16 @@ func (m *ByteOffsetPositionMapper) Position(byteOffset int) *storepb.Position {for m.byteOffset < byteOffset {r, size := utf8.DecodeRuneInString(m.sql[m.byteOffset:])- if r == '\n' {+ switch r {+ case '\r':m.line++m.runeCol = 0- } else {+ case '\n':+ if m.byteOffset == 0 || m.sql[m.byteOffset-1] != '\r' {+ m.line+++ m.runeCol = 0+ }+ default:m.runeCol++}m.byteOffset += size@@ -58,10 +64,16 @@ func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {i := 0for i < byteOffset {r, size := utf8.DecodeRuneInString(sql[i:])- if r == '\n' {+ switch r {+ case '\r':line++runeCol = 0- } else {+ case '\n':+ if i == 0 || sql[i-1] != '\r' {+ line+++ runeCol = 0+ }+ default:runeCol++}i += sizediff --git a/backend/plugin/parser/base/position_test.go b/backend/plugin/parser/base/position_test.goindex d28ef0ff07..986c5a6e14 100644--- a/backend/plugin/parser/base/position_test.go+++ b/backend/plugin/parser/base/position_test.go@@ -22,3 +22,43 @@ func TestByteOffsetPositionMapper(t *testing.T) {require.Equal(t, int32(1), mapper.Position(len("SELECT")).Line)require.Equal(t, int32(7), mapper.Position(len("SELECT")).Column)}++func TestByteOffsetPositionMapperLineBreaksMatchCalculateLineAndColumn(t *testing.T) {+ tests := []struct {+ name string+ sql string+ }{+ {+ name: "lf",+ sql: "SELECT 1;\nSELECT 日本;",+ },+ {+ name: "crlf",+ sql: "SELECT 1;\r\nSELECT 日本;",+ },+ {+ name: "cr",+ sql: "SELECT 1;\rSELECT 日本;",+ },+ {+ name: "mixed",+ sql: "SELECT 1;\r\nSELECT 日本;\rSELECT 3;\nSELECT 4;",+ },+ }++ for _, test := range tests {+ t.Run(test.name, func(t *testing.T) {+ mapper := NewByteOffsetPositionMapper(test.sql)+ offsets := []int{len(test.sql)}+ for offset := range test.sql {+ offsets = append(offsets, offset)+ }+ for _, offset := range offsets {+ line, column := CalculateLineAndColumn(test.sql, offset)+ got := mapper.Position(offset)+ require.Equal(t, int32(line+1), got.Line, "offset %d", offset)+ require.Equal(t, int32(column+1), got.Column, "offset %d", offset)+ }+ })+ }+}diff --git a/backend/plugin/parser/cosmosdb/cosmosdb.go b/backend/plugin/parser/cosmosdb/cosmosdb.goindex 8c6ba17238..861d061b86 100644--- a/backend/plugin/parser/cosmosdb/cosmosdb.go+++ b/backend/plugin/parser/cosmosdb/cosmosdb.go@@ -1,8 +1,6 @@package cosmosdbimport (- "unicode/utf8"-omnicosmosdb "github.com/bytebase/omni/cosmosdb""github.com/bytebase/omni/cosmosdb/ast"@@ -42,6 +40,7 @@ func ParseCosmosDB(statement string) ([]*OmniAST, error) {}var result []*OmniAST+ positionMapper := base.NewByteOffsetPositionMapper(statement)for _, stmt := range stmts {if stmt.Empty() {continue@@ -49,35 +48,8 @@ func ParseCosmosDB(statement string) ([]*OmniAST, error) {result = append(result, &OmniAST{Node: stmt.AST,Text: stmt.Text,- StartPosition: byteOffsetToRunePosition(statement, stmt.ByteStart),+ StartPosition: positionMapper.Position(stmt.ByteStart),})}return result, nil}--// byteOffsetToRunePosition converts a byte offset to a 1-based line:column position-// where column is measured in Unicode code points.-func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {- if byteOffset > len(sql) {- byteOffset = len(sql)- }-- line := int32(1)- runeCol := int32(0)- i := 0- for i < byteOffset {- r, size := utf8.DecodeRuneInString(sql[i:])- if r == '\n' {- line++- runeCol = 0- } else {- runeCol++- }- i += size- }-- return &storepb.Position{- Line: line,- Column: runeCol + 1,- }-}diff --git a/backend/plugin/parser/cosmosdb/split.go b/backend/plugin/parser/cosmosdb/split.goindex 4a2781f7c0..55c72b761d 100644--- a/backend/plugin/parser/cosmosdb/split.go+++ b/backend/plugin/parser/cosmosdb/split.go@@ -29,9 +29,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {}var result []base.Statement+ positionMapper := base.NewByteOffsetPositionMapper(statement)for _, stmt := range stmts {- startPos := byteOffsetToRunePosition(statement, stmt.ByteStart)- endPos := byteOffsetToRunePosition(statement, stmt.ByteEnd)+ startPos := positionMapper.Position(stmt.ByteStart)+ endPos := positionMapper.Position(stmt.ByteEnd)result = append(result, base.Statement{Text: stmt.Text,diff --git a/backend/plugin/parser/mongodb/mongodb.go b/backend/plugin/parser/mongodb/mongodb.goindex 9b0bd2236c..f51e98f7a4 100644--- a/backend/plugin/parser/mongodb/mongodb.go+++ b/backend/plugin/parser/mongodb/mongodb.go@@ -1,8 +1,6 @@package mongodbimport (- "unicode/utf8"-"github.com/bytebase/omni/mongo""github.com/bytebase/omni/mongo/ast"omniparser "github.com/bytebase/omni/mongo/parser"@@ -63,12 +61,13 @@ func ParseMongoShellBestEffort(statement string) ([]base.ParsedStatement, []*omnfunc convertStatements(input string, stmts []mongo.Statement) []base.ParsedStatement {var result []base.ParsedStatement+ positionMapper := base.NewByteOffsetPositionMapper(input)for _, stmt := range stmts {if stmt.Empty() {continue}- startPos := byteOffsetToPosition(input, stmt.ByteStart)- endPos := byteOffsetToPosition(input, stmt.ByteEnd)+ startPos := positionMapper.Position(stmt.ByteStart)+ endPos := positionMapper.Position(stmt.ByteEnd)result = append(result, base.ParsedStatement{Statement: base.Statement{Text: stmt.Text,@@ -87,30 +86,3 @@ func convertStatements(input string, stmts []mongo.Statement) []base.ParsedState}return result}--// byteOffsetToPosition converts a byte offset to a 1-based line:column position-// where column is measured in Unicode code points.-func byteOffsetToPosition(sql string, byteOffset int) *storepb.Position {- if byteOffset > len(sql) {- byteOffset = len(sql)- }-- line := int32(1)- runeCol := int32(0)- i := 0- for i < byteOffset {- r, size := utf8.DecodeRuneInString(sql[i:])- if r == '\n' {- line++- runeCol = 0- } else {- runeCol++- }- i += size- }-- return &storepb.Position{- Line: line,- Column: runeCol + 1,- }-}diff --git a/backend/plugin/parser/redshift/split.go b/backend/plugin/parser/redshift/split.goindex 6640e29566..72f3078149 100644--- a/backend/plugin/parser/redshift/split.go+++ b/backend/plugin/parser/redshift/split.go@@ -15,20 +15,13 @@ func init() {func SplitSQL(statement string) ([]base.Statement, error) {segments := omniredshift.Split(statement)list := make([]base.Statement, 0, len(segments))+ positionMapper := base.NewByteOffsetPositionMapper(statement)for _, segment := range segments {- startLine, startColumn := base.CalculateLineAndColumn(statement, segment.ByteStart)- endLine, endColumn := base.CalculateLineAndColumn(statement, segment.ByteEnd)list = append(list, base.Statement{Text: segment.Text,Empty: segment.Empty(),- Start: &storepb.Position{- Line: int32(startLine + 1),- Column: int32(startColumn + 1),- },- End: &storepb.Position{- Line: int32(endLine + 1),- Column: int32(endColumn + 1),- },+ Start: positionMapper.Position(segment.ByteStart),+ End: positionMapper.Position(segment.ByteEnd),Range: &storepb.Range{Start: int32(segment.ByteStart),End: int32(segment.ByteEnd),diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.goindex fbf614d025..9c1b44b7b6 100644--- a/backend/plugin/parser/tsql/split.go+++ b/backend/plugin/parser/tsql/split.go@@ -55,6 +55,7 @@ func toBaseStatements(sql string, omniStmts []omnimssql.Statement) []base.Statemreturn nil}result := make([]base.Statement, 0, len(omniStmts))+ positionMapper := base.NewByteOffsetPositionMapper(sql)prevEnd := &storepb.Position{Line: 1, Column: 1}prevByte := 0for _, os := range omniStmts {@@ -64,11 +65,7 @@ func toBaseStatements(sql string, omniStmts []omnimssql.Statement) []base.StatemendByte = startByte}text := sql[startByte:endByte]- endLine, endCol := base.CalculateLineAndColumn(sql, endByte)- endPos := &storepb.Position{- Line: int32(endLine + 1),- Column: int32(endCol + 1),- }+ endPos := positionMapper.Position(endByte)_, isGo := os.AST.(*ast.GoStmt)empty := isGo || os.Empty() || strings.TrimSpace(text) == ""