Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(parser): make omni split position mapping linear (#20430)

rebelicedecfe22ffc0d
diff --git a/backend/plugin/parser/base/position.go b/backend/plugin/parser/base/position.go
new file mode 100644
index 0000000000..e298e668cb
--- /dev/null
+++ b/backend/plugin/parser/base/position.go
@@ -0,0 +1,74 @@
+package base
+
+import (
+ "unicode/utf8"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+// ByteOffsetPositionMapper converts monotonically increasing byte offsets in a
+// single SQL string to 1-based line:column positions in one pass.
+type ByteOffsetPositionMapper struct {
+ sql string
+ byteOffset int
+ line int32
+ runeCol int32
+}
+
+// NewByteOffsetPositionMapper creates a mapper for byte offsets in sql.
+func NewByteOffsetPositionMapper(sql string) *ByteOffsetPositionMapper {
+ return &ByteOffsetPositionMapper{
+ sql: sql,
+ line: 1,
+ }
+}
+
+// Position returns the 1-based line:column position for byteOffset.
+func (m *ByteOffsetPositionMapper) Position(byteOffset int) *storepb.Position {
+ if byteOffset < 0 {
+ byteOffset = 0
+ }
+ if byteOffset > len(m.sql) {
+ byteOffset = len(m.sql)
+ }
+ if byteOffset < m.byteOffset {
+ return byteOffsetToRunePosition(m.sql, byteOffset)
+ }
+
+ for m.byteOffset < byteOffset {
+ r, size := utf8.DecodeRuneInString(m.sql[m.byteOffset:])
+ if r == '\n' {
+ m.line++
+ m.runeCol = 0
+ } else {
+ m.runeCol++
+ }
+ m.byteOffset += size
+ }
+
+ return &storepb.Position{
+ Line: m.line,
+ Column: m.runeCol + 1,
+ }
+}
+
+func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {
+ line := int32(1)
+ runeCol := int32(0)
+ i := 0
+ for i < byteOffset {
+ r, size := utf8.DecodeRuneInString(sql[i:])
+ if r == '\n' {
+ line++
+ runeCol = 0
+ } else {
+ runeCol++
+ }
+ i += size
+ }
+
+ return &storepb.Position{
+ Line: line,
+ Column: runeCol + 1,
+ }
+}
diff --git a/backend/plugin/parser/base/position_test.go b/backend/plugin/parser/base/position_test.go
new file mode 100644
index 0000000000..d28ef0ff07
--- /dev/null
+++ b/backend/plugin/parser/base/position_test.go
@@ -0,0 +1,24 @@
+package base
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestByteOffsetPositionMapper(t *testing.T) {
+ sql := "SELECT 1;\nSELECT 日本;\nSELECT 3;"
+ mapper := NewByteOffsetPositionMapper(sql)
+
+ require.Equal(t, int32(1), mapper.Position(0).Line)
+ require.Equal(t, int32(1), mapper.Position(0).Column)
+ require.Equal(t, int32(2), mapper.Position(len("SELECT 1;\n")).Line)
+ require.Equal(t, int32(1), mapper.Position(len("SELECT 1;\n")).Column)
+ require.Equal(t, int32(2), mapper.Position(len("SELECT 1;\nSELECT 日")).Line)
+ require.Equal(t, int32(9), mapper.Position(len("SELECT 1;\nSELECT 日")).Column)
+
+ // Out-of-order lookups are supported for callers that occasionally need to
+ // revisit an earlier offset.
+ require.Equal(t, int32(1), mapper.Position(len("SELECT")).Line)
+ require.Equal(t, int32(7), mapper.Position(len("SELECT")).Column)
+}
diff --git a/backend/plugin/parser/mysql/split.go b/backend/plugin/parser/mysql/split.go
index c7fc141411..5ca5ac9c05 100644
--- a/backend/plugin/parser/mysql/split.go
+++ b/backend/plugin/parser/mysql/split.go
@@ -21,6 +21,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {
segments := mysqlparser.Split(statement)
result := make([]base.Statement, 0, len(segments))
+ positionMapper := base.NewByteOffsetPositionMapper(statement)
for _, seg := range segments {
// omni Segment excludes the trailing semicolon, but downstream
// code expects it included. Extend the byte range to cover it.
@@ -33,8 +34,8 @@ func SplitSQL(statement string) ([]base.Statement, error) {
result = append(result, base.Statement{
Text: text,
Empty: seg.Empty(),
- Start: ByteOffsetToRunePosition(statement, seg.ByteStart),
- End: ByteOffsetToRunePosition(statement, byteEnd),
+ Start: positionMapper.Position(seg.ByteStart),
+ End: positionMapper.Position(byteEnd),
Range: &storepb.Range{
Start: int32(seg.ByteStart),
End: int32(byteEnd),
diff --git a/backend/plugin/parser/mysql/split_test.go b/backend/plugin/parser/mysql/split_test.go
index 2aa8411291..d566b3fdec 100644
--- a/backend/plugin/parser/mysql/split_test.go
+++ b/backend/plugin/parser/mysql/split_test.go
@@ -1,7 +1,10 @@
package mysql
import (
+ "fmt"
+ "strings"
"testing"
+ "time"
"github.com/stretchr/testify/require"
@@ -14,6 +17,23 @@ func TestMySQLSplitSQL(t *testing.T) {
})
}
+func TestMySQLSplitSQLLargeInsertScriptScalesLinearly(t *testing.T) {
+ const rowCount = 2000
+ padding := strings.Repeat("x", 1024)
+ var builder strings.Builder
+ for i := 0; i < rowCount; i++ {
+ fmt.Fprintf(&builder, "INSERT INTO perf_omni_mysql (id, payload) VALUES (%d, '%s');\n", i, padding)
+ }
+
+ started := time.Now()
+ statements, err := SplitSQL(builder.String())
+ elapsed := time.Since(started)
+
+ require.NoError(t, err)
+ require.Len(t, base.FilterEmptyStatements(statements), rowCount)
+ require.Less(t, elapsed, time.Second)
+}
+
func TestSplitMySQLStatements(t *testing.T) {
tests := []struct {
statement string
diff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.go
index 8728d8e1b9..6308e1a9cc 100644
--- a/backend/plugin/parser/pg/split.go
+++ b/backend/plugin/parser/pg/split.go
@@ -19,12 +19,13 @@ func SplitSQL(statement string) ([]base.Statement, error) {
segments := omnipg.Split(statement)
result := make([]base.Statement, 0, len(segments))
+ positionMapper := base.NewByteOffsetPositionMapper(statement)
for _, seg := range segments {
result = append(result, base.Statement{
Text: seg.Text,
Empty: seg.Empty(),
- Start: ByteOffsetToRunePosition(statement, seg.ByteStart),
- End: ByteOffsetToRunePosition(statement, seg.ByteEnd),
+ Start: positionMapper.Position(seg.ByteStart),
+ End: positionMapper.Position(seg.ByteEnd),
Range: &storepb.Range{
Start: int32(seg.ByteStart),
End: int32(seg.ByteEnd),
diff --git a/backend/plugin/parser/pg/split_test.go b/backend/plugin/parser/pg/split_test.go
index 5e8fba118a..1a4e71f815 100644
--- a/backend/plugin/parser/pg/split_test.go
+++ b/backend/plugin/parser/pg/split_test.go
@@ -1,7 +1,12 @@
package pg
import (
+ "fmt"
+ "strings"
"testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -11,3 +16,20 @@ func TestPGSplitSQL(t *testing.T) {
SplitFunc: SplitSQL,
})
}
+
+func TestPGSplitSQLLargeInsertScriptScalesLinearly(t *testing.T) {
+ const rowCount = 2000
+ padding := strings.Repeat("x", 1024)
+ var builder strings.Builder
+ for i := 0; i < rowCount; i++ {
+ fmt.Fprintf(&builder, "INSERT INTO perf_omni_pg (id, payload) VALUES (%d, '%s');\n", i, padding)
+ }
+
+ started := time.Now()
+ statements, err := SplitSQL(builder.String())
+ elapsed := time.Since(started)
+
+ require.NoError(t, err)
+ require.Len(t, base.FilterEmptyStatements(statements), rowCount)
+ require.Less(t, elapsed, time.Second)
+}
diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.go
index 749394705b..7e59badcbc 100644
--- a/backend/plugin/parser/plsql/split.go
+++ b/backend/plugin/parser/plsql/split.go
@@ -40,6 +40,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {
segments := oracleparser.Split(statement)
result := make([]base.Statement, 0, len(segments))
+ positionMapper := base.NewByteOffsetPositionMapper(statement)
for _, seg := range segments {
if seg.Kind == oracleparser.SegmentSQLPlusCommand {
continue
@@ -48,8 +49,8 @@ func SplitSQL(statement string) ([]base.Statement, error) {
text := statement[seg.ByteStart:byteEnd]
result = append(result, base.Statement{
Text: text,
- Start: ByteOffsetToRunePosition(statement, seg.ByteStart),
- End: ByteOffsetToRunePosition(statement, byteEnd),
+ Start: positionMapper.Position(seg.ByteStart),
+ End: positionMapper.Position(byteEnd),
Empty: seg.Empty(),
Range: &storepb.Range{
Start: int32(seg.ByteStart),
diff --git a/backend/plugin/parser/plsql/split_test.go b/backend/plugin/parser/plsql/split_test.go
index b94a83b15a..76515dccd0 100644
--- a/backend/plugin/parser/plsql/split_test.go
+++ b/backend/plugin/parser/plsql/split_test.go
@@ -1,7 +1,10 @@
package plsql
import (
+ "fmt"
+ "strings"
"testing"
+ "time"
"github.com/stretchr/testify/require"
@@ -66,3 +69,23 @@ SELECT 1 FROM dual;`
require.Equal(t, "\nSET ROLE app_role", statements[1].Text)
require.Equal(t, "\nSELECT 1 FROM dual", statements[2].Text)
}
+
+func TestPLSQLSplitSQLLargeInsertScriptScalesLinearly(t *testing.T) {
+ const rowCount = 2000
+ padding := strings.Repeat("x", 1024)
+ var builder strings.Builder
+ for i := 0; i < rowCount; i++ {
+ fmt.Fprintf(&builder, "INSERT INTO perf_omni_oracle (id, payload) VALUES (%d, '%s');\n", i, padding)
+ }
+
+ started := time.Now()
+ statements, err := SplitSQL(builder.String())
+ elapsed := time.Since(started)
+
+ require.NoError(t, err)
+ require.Len(t, statements, rowCount)
+ require.Less(t, elapsed, time.Second)
+ require.Equal(t, int32(1), statements[0].Start.Line)
+ require.Equal(t, int32(rowCount-1), statements[rowCount-1].Start.Line)
+ require.Equal(t, int32(rowCount), statements[rowCount-1].End.Line)
+}