Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(parser): make omni split position mapping linear (#20430)
rebelicedecfe22ffc0d
diff --git a/backend/plugin/parser/base/position.go b/backend/plugin/parser/base/position.gonew file mode 100644index 0000000000..e298e668cb--- /dev/null+++ b/backend/plugin/parser/base/position.go@@ -0,0 +1,74 @@+package base++import (+ "unicode/utf8"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// ByteOffsetPositionMapper converts monotonically increasing byte offsets in a+// single SQL string to 1-based line:column positions in one pass.+type ByteOffsetPositionMapper struct {+ sql string+ byteOffset int+ line int32+ runeCol int32+}++// NewByteOffsetPositionMapper creates a mapper for byte offsets in sql.+func NewByteOffsetPositionMapper(sql string) *ByteOffsetPositionMapper {+ return &ByteOffsetPositionMapper{+ sql: sql,+ line: 1,+ }+}++// Position returns the 1-based line:column position for byteOffset.+func (m *ByteOffsetPositionMapper) Position(byteOffset int) *storepb.Position {+ if byteOffset < 0 {+ byteOffset = 0+ }+ if byteOffset > len(m.sql) {+ byteOffset = len(m.sql)+ }+ if byteOffset < m.byteOffset {+ return byteOffsetToRunePosition(m.sql, byteOffset)+ }++ for m.byteOffset < byteOffset {+ r, size := utf8.DecodeRuneInString(m.sql[m.byteOffset:])+ if r == '\n' {+ m.line+++ m.runeCol = 0+ } else {+ m.runeCol+++ }+ m.byteOffset += size+ }++ return &storepb.Position{+ Line: m.line,+ Column: m.runeCol + 1,+ }+}++func byteOffsetToRunePosition(sql string, byteOffset int) *storepb.Position {+ line := int32(1)+ runeCol := int32(0)+ i := 0+ for i < byteOffset {+ r, size := utf8.DecodeRuneInString(sql[i:])+ if r == '\n' {+ line+++ runeCol = 0+ } else {+ runeCol+++ }+ i += size+ }++ return &storepb.Position{+ Line: line,+ Column: runeCol + 1,+ }+}diff --git a/backend/plugin/parser/base/position_test.go b/backend/plugin/parser/base/position_test.gonew file mode 100644index 0000000000..d28ef0ff07--- /dev/null+++ b/backend/plugin/parser/base/position_test.go@@ -0,0 +1,24 @@+package base++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestByteOffsetPositionMapper(t *testing.T) {+ sql := "SELECT 1;\nSELECT 日本;\nSELECT 3;"+ mapper := NewByteOffsetPositionMapper(sql)++ require.Equal(t, int32(1), mapper.Position(0).Line)+ require.Equal(t, int32(1), mapper.Position(0).Column)+ require.Equal(t, int32(2), mapper.Position(len("SELECT 1;\n")).Line)+ require.Equal(t, int32(1), mapper.Position(len("SELECT 1;\n")).Column)+ require.Equal(t, int32(2), mapper.Position(len("SELECT 1;\nSELECT 日")).Line)+ require.Equal(t, int32(9), mapper.Position(len("SELECT 1;\nSELECT 日")).Column)++ // Out-of-order lookups are supported for callers that occasionally need to+ // revisit an earlier offset.+ require.Equal(t, int32(1), mapper.Position(len("SELECT")).Line)+ require.Equal(t, int32(7), mapper.Position(len("SELECT")).Column)+}diff --git a/backend/plugin/parser/mysql/split.go b/backend/plugin/parser/mysql/split.goindex c7fc141411..5ca5ac9c05 100644--- a/backend/plugin/parser/mysql/split.go+++ b/backend/plugin/parser/mysql/split.go@@ -21,6 +21,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {segments := mysqlparser.Split(statement)result := make([]base.Statement, 0, len(segments))+ positionMapper := base.NewByteOffsetPositionMapper(statement)for _, seg := range segments {// omni Segment excludes the trailing semicolon, but downstream// code expects it included. Extend the byte range to cover it.@@ -33,8 +34,8 @@ func SplitSQL(statement string) ([]base.Statement, error) {result = append(result, base.Statement{Text: text,Empty: seg.Empty(),- Start: ByteOffsetToRunePosition(statement, seg.ByteStart),- End: ByteOffsetToRunePosition(statement, byteEnd),+ Start: positionMapper.Position(seg.ByteStart),+ End: positionMapper.Position(byteEnd),Range: &storepb.Range{Start: int32(seg.ByteStart),End: int32(byteEnd),diff --git a/backend/plugin/parser/mysql/split_test.go b/backend/plugin/parser/mysql/split_test.goindex 2aa8411291..d566b3fdec 100644--- a/backend/plugin/parser/mysql/split_test.go+++ b/backend/plugin/parser/mysql/split_test.go@@ -1,7 +1,10 @@package mysqlimport (+ "fmt"+ "strings""testing"+ "time""github.com/stretchr/testify/require"@@ -14,6 +17,23 @@ func TestMySQLSplitSQL(t *testing.T) {})}+func TestMySQLSplitSQLLargeInsertScriptScalesLinearly(t *testing.T) {+ const rowCount = 2000+ padding := strings.Repeat("x", 1024)+ var builder strings.Builder+ for i := 0; i < rowCount; i++ {+ fmt.Fprintf(&builder, "INSERT INTO perf_omni_mysql (id, payload) VALUES (%d, '%s');\n", i, padding)+ }++ started := time.Now()+ statements, err := SplitSQL(builder.String())+ elapsed := time.Since(started)++ require.NoError(t, err)+ require.Len(t, base.FilterEmptyStatements(statements), rowCount)+ require.Less(t, elapsed, time.Second)+}+func TestSplitMySQLStatements(t *testing.T) {tests := []struct {statement stringdiff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.goindex 8728d8e1b9..6308e1a9cc 100644--- a/backend/plugin/parser/pg/split.go+++ b/backend/plugin/parser/pg/split.go@@ -19,12 +19,13 @@ func SplitSQL(statement string) ([]base.Statement, error) {segments := omnipg.Split(statement)result := make([]base.Statement, 0, len(segments))+ positionMapper := base.NewByteOffsetPositionMapper(statement)for _, seg := range segments {result = append(result, base.Statement{Text: seg.Text,Empty: seg.Empty(),- Start: ByteOffsetToRunePosition(statement, seg.ByteStart),- End: ByteOffsetToRunePosition(statement, seg.ByteEnd),+ Start: positionMapper.Position(seg.ByteStart),+ End: positionMapper.Position(seg.ByteEnd),Range: &storepb.Range{Start: int32(seg.ByteStart),End: int32(seg.ByteEnd),diff --git a/backend/plugin/parser/pg/split_test.go b/backend/plugin/parser/pg/split_test.goindex 5e8fba118a..1a4e71f815 100644--- a/backend/plugin/parser/pg/split_test.go+++ b/backend/plugin/parser/pg/split_test.go@@ -1,7 +1,12 @@package pgimport (+ "fmt"+ "strings""testing"+ "time"++ "github.com/stretchr/testify/require""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -11,3 +16,20 @@ func TestPGSplitSQL(t *testing.T) {SplitFunc: SplitSQL,})}++func TestPGSplitSQLLargeInsertScriptScalesLinearly(t *testing.T) {+ const rowCount = 2000+ padding := strings.Repeat("x", 1024)+ var builder strings.Builder+ for i := 0; i < rowCount; i++ {+ fmt.Fprintf(&builder, "INSERT INTO perf_omni_pg (id, payload) VALUES (%d, '%s');\n", i, padding)+ }++ started := time.Now()+ statements, err := SplitSQL(builder.String())+ elapsed := time.Since(started)++ require.NoError(t, err)+ require.Len(t, base.FilterEmptyStatements(statements), rowCount)+ require.Less(t, elapsed, time.Second)+}diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.goindex 749394705b..7e59badcbc 100644--- a/backend/plugin/parser/plsql/split.go+++ b/backend/plugin/parser/plsql/split.go@@ -40,6 +40,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {segments := oracleparser.Split(statement)result := make([]base.Statement, 0, len(segments))+ positionMapper := base.NewByteOffsetPositionMapper(statement)for _, seg := range segments {if seg.Kind == oracleparser.SegmentSQLPlusCommand {continue@@ -48,8 +49,8 @@ func SplitSQL(statement string) ([]base.Statement, error) {text := statement[seg.ByteStart:byteEnd]result = append(result, base.Statement{Text: text,- Start: ByteOffsetToRunePosition(statement, seg.ByteStart),- End: ByteOffsetToRunePosition(statement, byteEnd),+ Start: positionMapper.Position(seg.ByteStart),+ End: positionMapper.Position(byteEnd),Empty: seg.Empty(),Range: &storepb.Range{Start: int32(seg.ByteStart),diff --git a/backend/plugin/parser/plsql/split_test.go b/backend/plugin/parser/plsql/split_test.goindex b94a83b15a..76515dccd0 100644--- a/backend/plugin/parser/plsql/split_test.go+++ b/backend/plugin/parser/plsql/split_test.go@@ -1,7 +1,10 @@package plsqlimport (+ "fmt"+ "strings""testing"+ "time""github.com/stretchr/testify/require"@@ -66,3 +69,23 @@ SELECT 1 FROM dual;`require.Equal(t, "\nSET ROLE app_role", statements[1].Text)require.Equal(t, "\nSELECT 1 FROM dual", statements[2].Text)}++func TestPLSQLSplitSQLLargeInsertScriptScalesLinearly(t *testing.T) {+ const rowCount = 2000+ padding := strings.Repeat("x", 1024)+ var builder strings.Builder+ for i := 0; i < rowCount; i++ {+ fmt.Fprintf(&builder, "INSERT INTO perf_omni_oracle (id, payload) VALUES (%d, '%s');\n", i, padding)+ }++ started := time.Now()+ statements, err := SplitSQL(builder.String())+ elapsed := time.Since(started)++ require.NoError(t, err)+ require.Len(t, statements, rowCount)+ require.Less(t, elapsed, time.Second)+ require.Equal(t, int32(1), statements[0].Start.Line)+ require.Equal(t, int32(rowCount-1), statements[rowCount-1].Start.Line)+ require.Equal(t, int32(rowCount), statements[rowCount-1].End.Line)+}