Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix: make omni split position mapping linear (#20556)

rebelice7585603766af
diff --git a/backend/plugin/parser/base/position.go b/backend/plugin/parser/base/position.go
index e298e668cb..d9884409a6 100644
--- a/backend/plugin/parser/base/position.go
+++ b/backend/plugin/parser/base/position.go
@@ -37,10 +37,16 @@ func (m *ByteOffsetPositionMapper) Position(byteOffset int) *storepb.Position {
for m.byteOffset < byteOffset {
r, size := utf8.DecodeRuneInString(m.sql[m.byteOffset:])
- if r == '\n' {
+ switch r {
+ case '\r':
m.line++
m.runeCol = 0
- } else {
+ case '\n':
+ if m.byteOffset == 0 || m.sql[m.byteOffset-1] != '\r' {
+ m.line++
+ m.runeCol = 0
+ }
+ default:
m.runeCol++
}
m.byteOffset += size
@@ -58,10 +64,16 @@ func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {
i := 0
for i < byteOffset {
r, size := utf8.DecodeRuneInString(sql[i:])
- if r == '\n' {
+ switch r {
+ case '\r':
line++
runeCol = 0
- } else {
+ case '\n':
+ if i == 0 || sql[i-1] != '\r' {
+ line++
+ runeCol = 0
+ }
+ default:
runeCol++
}
i += size
diff --git a/backend/plugin/parser/base/position_test.go b/backend/plugin/parser/base/position_test.go
index d28ef0ff07..986c5a6e14 100644
--- a/backend/plugin/parser/base/position_test.go
+++ b/backend/plugin/parser/base/position_test.go
@@ -22,3 +22,43 @@ func TestByteOffsetPositionMapper(t *testing.T) {
require.Equal(t, int32(1), mapper.Position(len("SELECT")).Line)
require.Equal(t, int32(7), mapper.Position(len("SELECT")).Column)
}
+
+func TestByteOffsetPositionMapperLineBreaksMatchCalculateLineAndColumn(t *testing.T) {
+ tests := []struct {
+ name string
+ sql string
+ }{
+ {
+ name: "lf",
+ sql: "SELECT 1;\nSELECT 日本;",
+ },
+ {
+ name: "crlf",
+ sql: "SELECT 1;\r\nSELECT 日本;",
+ },
+ {
+ name: "cr",
+ sql: "SELECT 1;\rSELECT 日本;",
+ },
+ {
+ name: "mixed",
+ sql: "SELECT 1;\r\nSELECT 日本;\rSELECT 3;\nSELECT 4;",
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ mapper := NewByteOffsetPositionMapper(test.sql)
+ offsets := []int{len(test.sql)}
+ for offset := range test.sql {
+ offsets = append(offsets, offset)
+ }
+ for _, offset := range offsets {
+ line, column := CalculateLineAndColumn(test.sql, offset)
+ got := mapper.Position(offset)
+ require.Equal(t, int32(line+1), got.Line, "offset %d", offset)
+ require.Equal(t, int32(column+1), got.Column, "offset %d", offset)
+ }
+ })
+ }
+}
diff --git a/backend/plugin/parser/cosmosdb/cosmosdb.go b/backend/plugin/parser/cosmosdb/cosmosdb.go
index 8c6ba17238..861d061b86 100644
--- a/backend/plugin/parser/cosmosdb/cosmosdb.go
+++ b/backend/plugin/parser/cosmosdb/cosmosdb.go
@@ -1,8 +1,6 @@
package cosmosdb
import (
- "unicode/utf8"
-
omnicosmosdb "github.com/bytebase/omni/cosmosdb"
"github.com/bytebase/omni/cosmosdb/ast"
@@ -42,6 +40,7 @@ func ParseCosmosDB(statement string) ([]*OmniAST, error) {
}
var result []*OmniAST
+ positionMapper := base.NewByteOffsetPositionMapper(statement)
for _, stmt := range stmts {
if stmt.Empty() {
continue
@@ -49,35 +48,8 @@ func ParseCosmosDB(statement string) ([]*OmniAST, error) {
result = append(result, &OmniAST{
Node: stmt.AST,
Text: stmt.Text,
- StartPosition: byteOffsetToRunePosition(statement, stmt.ByteStart),
+ StartPosition: positionMapper.Position(stmt.ByteStart),
})
}
return result, nil
}
-
-// byteOffsetToRunePosition converts a byte offset to a 1-based line:column position
-// where column is measured in Unicode code points.
-func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {
- if byteOffset > len(sql) {
- byteOffset = len(sql)
- }
-
- line := int32(1)
- runeCol := int32(0)
- i := 0
- for i < byteOffset {
- r, size := utf8.DecodeRuneInString(sql[i:])
- if r == '\n' {
- line++
- runeCol = 0
- } else {
- runeCol++
- }
- i += size
- }
-
- return &storepb.Position{
- Line: line,
- Column: runeCol + 1,
- }
-}
diff --git a/backend/plugin/parser/cosmosdb/split.go b/backend/plugin/parser/cosmosdb/split.go
index 4a2781f7c0..55c72b761d 100644
--- a/backend/plugin/parser/cosmosdb/split.go
+++ b/backend/plugin/parser/cosmosdb/split.go
@@ -29,9 +29,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {
}
var result []base.Statement
+ positionMapper := base.NewByteOffsetPositionMapper(statement)
for _, stmt := range stmts {
- startPos := byteOffsetToRunePosition(statement, stmt.ByteStart)
- endPos := byteOffsetToRunePosition(statement, stmt.ByteEnd)
+ startPos := positionMapper.Position(stmt.ByteStart)
+ endPos := positionMapper.Position(stmt.ByteEnd)
result = append(result, base.Statement{
Text: stmt.Text,
diff --git a/backend/plugin/parser/mongodb/mongodb.go b/backend/plugin/parser/mongodb/mongodb.go
index 9b0bd2236c..f51e98f7a4 100644
--- a/backend/plugin/parser/mongodb/mongodb.go
+++ b/backend/plugin/parser/mongodb/mongodb.go
@@ -1,8 +1,6 @@
package mongodb
import (
- "unicode/utf8"
-
"github.com/bytebase/omni/mongo"
"github.com/bytebase/omni/mongo/ast"
omniparser "github.com/bytebase/omni/mongo/parser"
@@ -63,12 +61,13 @@ func ParseMongoShellBestEffort(statement string) ([]base.ParsedStatement, []*omn
func convertStatements(input string, stmts []mongo.Statement) []base.ParsedStatement {
var result []base.ParsedStatement
+ positionMapper := base.NewByteOffsetPositionMapper(input)
for _, stmt := range stmts {
if stmt.Empty() {
continue
}
- startPos := byteOffsetToPosition(input, stmt.ByteStart)
- endPos := byteOffsetToPosition(input, stmt.ByteEnd)
+ startPos := positionMapper.Position(stmt.ByteStart)
+ endPos := positionMapper.Position(stmt.ByteEnd)
result = append(result, base.ParsedStatement{
Statement: base.Statement{
Text: stmt.Text,
@@ -87,30 +86,3 @@ func convertStatements(input string, stmts []mongo.Statement) []base.ParsedState
}
return result
}
-
-// byteOffsetToPosition converts a byte offset to a 1-based line:column position
-// where column is measured in Unicode code points.
-func byteOffsetToPosition(sql string, byteOffset int) *storepb.Position {
- if byteOffset > len(sql) {
- byteOffset = len(sql)
- }
-
- line := int32(1)
- runeCol := int32(0)
- i := 0
- for i < byteOffset {
- r, size := utf8.DecodeRuneInString(sql[i:])
- if r == '\n' {
- line++
- runeCol = 0
- } else {
- runeCol++
- }
- i += size
- }
-
- return &storepb.Position{
- Line: line,
- Column: runeCol + 1,
- }
-}
diff --git a/backend/plugin/parser/redshift/split.go b/backend/plugin/parser/redshift/split.go
index 6640e29566..72f3078149 100644
--- a/backend/plugin/parser/redshift/split.go
+++ b/backend/plugin/parser/redshift/split.go
@@ -15,20 +15,13 @@ func init() {
func SplitSQL(statement string) ([]base.Statement, error) {
segments := omniredshift.Split(statement)
list := make([]base.Statement, 0, len(segments))
+ positionMapper := base.NewByteOffsetPositionMapper(statement)
for _, segment := range segments {
- startLine, startColumn := base.CalculateLineAndColumn(statement, segment.ByteStart)
- endLine, endColumn := base.CalculateLineAndColumn(statement, segment.ByteEnd)
list = append(list, base.Statement{
Text: segment.Text,
Empty: segment.Empty(),
- Start: &storepb.Position{
- Line: int32(startLine + 1),
- Column: int32(startColumn + 1),
- },
- End: &storepb.Position{
- Line: int32(endLine + 1),
- Column: int32(endColumn + 1),
- },
+ Start: positionMapper.Position(segment.ByteStart),
+ End: positionMapper.Position(segment.ByteEnd),
Range: &storepb.Range{
Start: int32(segment.ByteStart),
End: int32(segment.ByteEnd),
diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.go
index fbf614d025..9c1b44b7b6 100644
--- a/backend/plugin/parser/tsql/split.go
+++ b/backend/plugin/parser/tsql/split.go
@@ -55,6 +55,7 @@ func toBaseStatements(sql string, omniStmts []omnimssql.Statement) []base.Statem
return nil
}
result := make([]base.Statement, 0, len(omniStmts))
+ positionMapper := base.NewByteOffsetPositionMapper(sql)
prevEnd := &storepb.Position{Line: 1, Column: 1}
prevByte := 0
for _, os := range omniStmts {
@@ -64,11 +65,7 @@ func toBaseStatements(sql string, omniStmts []omnimssql.Statement) []base.Statem
endByte = startByte
}
text := sql[startByte:endByte]
- endLine, endCol := base.CalculateLineAndColumn(sql, endByte)
- endPos := &storepb.Position{
- Line: int32(endLine + 1),
- Column: int32(endCol + 1),
- }
+ endPos := positionMapper.Position(endByte)
_, isGo := os.AST.(*ast.GoStmt)
empty := isGo || os.Empty() || strings.TrimSpace(text) == ""