Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(parser): add Start field to TiDB and Standard split implementations (#18628)

h3n4ldbb02d2119ec
diff --git a/backend/plugin/parser/standard/split.go b/backend/plugin/parser/standard/split.go
index 9da972dd5a..43a76644e4 100644
--- a/backend/plugin/parser/standard/split.go
+++ b/backend/plugin/parser/standard/split.go
@@ -33,14 +33,27 @@ func SplitSQL(statement string) ([]base.Statement, error) {
}
endPos := startPos + len(sql)
+ // Calculate line and column for Start position
+ startLine, startColumn := calculateLineAndColumn(statement, startPos)
+ // Calculate line and column for End position
+ endLine, endColumn := calculateLineAndColumn(statement, endPos)
+
list = append(list, base.Statement{
- Text: sql,
+ Text: sql,
+ BaseLine: startLine,
+ Start: &storepb.Position{
+ Line: int32(startLine + 1), // 1-based
+ Column: int32(startColumn + 1), // 1-based per proto spec
+ },
+ End: &storepb.Position{
+ Line: int32(endLine + 1), // 1-based
+ Column: int32(endColumn + 1), // 1-based per proto spec
+ },
Range: &storepb.Range{
Start: int32(startPos),
End: int32(endPos),
},
- End: &storepb.Position{Line: 0, Column: 0},
- Empty: false,
+ Empty: isEmptySQL(sql),
})
byteOffset = endPos
return nil
@@ -48,6 +61,53 @@ func SplitSQL(statement string) ([]base.Statement, error) {
return list, err
}
+// calculateLineAndColumn calculates the 0-based line number and 0-based column (character offset)
+// for a given byte offset in the statement.
+func calculateLineAndColumn(statement string, byteOffset int) (line, column int) {
+ if byteOffset > len(statement) {
+ byteOffset = len(statement)
+ }
+ // Range over string iterates over runes (code points), not bytes
+ for _, r := range statement[:byteOffset] {
+ if r == '\n' {
+ line++
+ column = 0
+ } else {
+ column++
+ }
+ }
+ return line, column
+}
+
+// isEmptySQL checks if the SQL contains only whitespace and comments.
+func isEmptySQL(sql string) bool {
+ trimmed := strings.TrimSpace(sql)
+ if trimmed == "" {
+ return true
+ }
+ // Check if it's only comments
+ if strings.HasPrefix(trimmed, "--") || strings.HasPrefix(trimmed, "/*") {
+ // Simple heuristic: if after removing comment markers there's no SQL
+ lines := strings.Split(trimmed, "\n")
+ for _, line := range lines {
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue
+ }
+ if strings.HasPrefix(line, "--") {
+ continue
+ }
+ if strings.HasPrefix(line, "/*") && strings.HasSuffix(line, "*/") {
+ continue
+ }
+ // Found non-comment content
+ return false
+ }
+ return true
+ }
+ return false
+}
+
// applyMultiStatements will apply the split statements from scanner.
// This function only used for SQLite, snowflake.
// For MySQL and PostgreSQL, use parser.SplitSQL.
diff --git a/backend/plugin/parser/standard/split_test.go b/backend/plugin/parser/standard/split_test.go
index 33334b2bab..d17110ecd6 100644
--- a/backend/plugin/parser/standard/split_test.go
+++ b/backend/plugin/parser/standard/split_test.go
@@ -8,6 +8,9 @@ import (
"time"
"github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
)
func generateOneMBInsert() string {
@@ -67,3 +70,146 @@ func TestApplyMultiStatements(t *testing.T) {
require.Equal(t, test.total, total)
}
}
+
+func TestSplitSQL(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ want []base.Statement
+ }{
+ {
+ name: "simple single statement",
+ statement: "SELECT 1;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 10},
+ Range: &storepb.Range{Start: 0, End: 9},
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multi-line statement",
+ statement: "SELECT\n 1;",
+ want: []base.Statement{
+ {
+ Text: "SELECT\n 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 5},
+ Range: &storepb.Range{Start: 0, End: 11},
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multiple statements",
+ statement: "SELECT 1;\nSELECT 2;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 10},
+ Range: &storepb.Range{Start: 0, End: 9},
+ Empty: false,
+ },
+ {
+ Text: "SELECT 2;",
+ BaseLine: 1,
+ Start: &storepb.Position{Line: 2, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 10},
+ Range: &storepb.Range{Start: 10, End: 19},
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multi-byte characters - Chinese",
+ statement: "SELECT '中文';",
+ want: []base.Statement{
+ {
+ Text: "SELECT '中文';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ // Column is 1-based character offset: S(1) E(2) L(3) E(4) C(5) T(6) ' '(7) '(8) 中(9) 文(10) '(11) ;(12) = 12 chars, End is exclusive so 13
+ End: &storepb.Position{Line: 1, Column: 13},
+ Range: &storepb.Range{Start: 0, End: 16}, // byte length: 8 + 3 + 3 + 2 = 16
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multi-byte characters - emoji",
+ statement: "SELECT '🎉';",
+ want: []base.Statement{
+ {
+ Text: "SELECT '🎉';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ // Column is 1-based character offset: S(1) E(2) L(3) E(4) C(5) T(6) ' '(7) '(8) 🎉(9) '(10) ;(11) = 11 chars, End is exclusive so 12
+ End: &storepb.Position{Line: 1, Column: 12},
+ Range: &storepb.Range{Start: 0, End: 14}, // byte length: 8 + 4 + 2 = 14
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multi-byte on second line",
+ statement: "SELECT\n '中文';",
+ want: []base.Statement{
+ {
+ Text: "SELECT\n '中文';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ // Line 2 (1-based): ' '(1) ' '(2) '(3) 中(4) 文(5) '(6) ;(7) = 7 chars, End is exclusive so 8
+ End: &storepb.Position{Line: 2, Column: 8},
+ Range: &storepb.Range{Start: 0, End: 18}, // 7 + 3 + 3 + 3 + 2 = 18
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multiple statements with multi-byte on separate lines",
+ statement: "SELECT '中';\nSELECT '文';",
+ want: []base.Statement{
+ {
+ Text: "SELECT '中';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ // S(1) E(2) L(3) E(4) C(5) T(6) ' '(7) '(8) 中(9) '(10) ;(11) = 11 chars, End is exclusive so 12
+ End: &storepb.Position{Line: 1, Column: 12},
+ Range: &storepb.Range{Start: 0, End: 13}, // 8 + 3 + 2 = 13
+ Empty: false,
+ },
+ {
+ Text: "SELECT '文';",
+ BaseLine: 1,
+ Start: &storepb.Position{Line: 2, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 12},
+ Range: &storepb.Range{Start: 14, End: 27}, // starts after newline
+ Empty: false,
+ },
+ },
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := SplitSQL(tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, len(tc.want), len(got), "number of statements mismatch")
+ for i, want := range tc.want {
+ require.Equal(t, want.Text, got[i].Text, "Text mismatch at index %d", i)
+ require.Equal(t, want.BaseLine, got[i].BaseLine, "BaseLine mismatch at index %d", i)
+ require.Equal(t, want.Start, got[i].Start, "Start mismatch at index %d", i)
+ require.Equal(t, want.End, got[i].End, "End mismatch at index %d", i)
+ require.Equal(t, want.Range, got[i].Range, "Range mismatch at index %d", i)
+ require.Equal(t, want.Empty, got[i].Empty, "Empty mismatch at index %d", i)
+ }
+ })
+ }
+}
diff --git a/backend/plugin/parser/tidb/split_test.go b/backend/plugin/parser/tidb/split_test.go
new file mode 100644
index 0000000000..9581183fd4
--- /dev/null
+++ b/backend/plugin/parser/tidb/split_test.go
@@ -0,0 +1,165 @@
+package tidb
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func TestSplitSQL(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ want []base.Statement
+ }{
+ {
+ name: "simple single statement",
+ statement: "SELECT 1;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 10}, // After semicolon (1-based exclusive)
+ Range: &storepb.Range{Start: 0, End: 9},
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multi-line statement",
+ statement: "SELECT\n 1;",
+ want: []base.Statement{
+ {
+ Text: "SELECT\n 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 5}, // After semicolon on line 2
+ Range: &storepb.Range{Start: 0, End: 11},
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multiple statements",
+ statement: "SELECT 1;\nSELECT 2;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 10},
+ Range: &storepb.Range{Start: 0, End: 9},
+ Empty: false,
+ },
+ {
+ Text: "SELECT 2;",
+ BaseLine: 1,
+ Start: &storepb.Position{Line: 2, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 10},
+ Range: &storepb.Range{Start: 10, End: 19},
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multi-byte characters - Chinese",
+ statement: "SELECT '中文';",
+ want: []base.Statement{
+ {
+ Text: "SELECT '中文';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ // Column is 1-based exclusive: S(1)..;(12), after = 13
+ End: &storepb.Position{Line: 1, Column: 13},
+ Range: &storepb.Range{Start: 0, End: 16}, // byte length: 8 + 3 + 3 + 2 = 16
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multi-byte characters - emoji",
+ statement: "SELECT '🎉';",
+ want: []base.Statement{
+ {
+ Text: "SELECT '🎉';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ // Column is 1-based exclusive: S(1)..;(11), after = 12
+ End: &storepb.Position{Line: 1, Column: 12},
+ Range: &storepb.Range{Start: 0, End: 14}, // byte length: 8 + 4 + 2 = 14
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multi-byte on second line",
+ statement: "SELECT\n '中文';",
+ want: []base.Statement{
+ {
+ Text: "SELECT\n '中文';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 8}, // After semicolon on line 2
+ Range: &storepb.Range{Start: 0, End: 18}, // 7 + 3 + 3 + 3 + 2 = 18
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "multiple statements with multi-byte",
+ statement: "SELECT '中';\nSELECT '文';",
+ want: []base.Statement{
+ {
+ Text: "SELECT '中';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 12}, // After semicolon
+ Range: &storepb.Range{Start: 0, End: 13}, // 8 + 3 + 2 = 13
+ Empty: false,
+ },
+ {
+ Text: "SELECT '文';",
+ BaseLine: 1,
+ Start: &storepb.Position{Line: 2, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 12},
+ Range: &storepb.Range{Start: 14, End: 27},
+ Empty: false,
+ },
+ },
+ },
+ {
+ name: "statement with leading spaces and multi-byte",
+ statement: " SELECT '中';",
+ want: []base.Statement{
+ {
+ Text: "SELECT '中';",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 3}, // starts after 2 spaces (1-based: column 3)
+ End: &storepb.Position{Line: 1, Column: 14}, // 2 leading spaces + 11 chars + 1 = 14
+ Range: &storepb.Range{Start: 2, End: 15}, // 2 + 8 + 3 + 2 = 15
+ Empty: false,
+ },
+ },
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := SplitSQL(tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, len(tc.want), len(got), "number of statements mismatch")
+ for i, want := range tc.want {
+ require.Equal(t, want.Text, got[i].Text, "Text mismatch at index %d", i)
+ require.Equal(t, want.BaseLine, got[i].BaseLine, "BaseLine mismatch at index %d", i)
+ require.Equal(t, want.Start, got[i].Start, "Start mismatch at index %d", i)
+ require.Equal(t, want.End, got[i].End, "End mismatch at index %d", i)
+ require.Equal(t, want.Range, got[i].Range, "Range mismatch at index %d", i)
+ require.Equal(t, want.Empty, got[i].Empty, "Empty mismatch at index %d", i)
+ }
+ })
+ }
+}
diff --git a/backend/plugin/parser/tokenizer/tokenizer.go b/backend/plugin/parser/tokenizer/tokenizer.go
index 434a445157..125896b375 100644
--- a/backend/plugin/parser/tokenizer/tokenizer.go
+++ b/backend/plugin/parser/tokenizer/tokenizer.go
@@ -221,7 +221,8 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
t.skipBlank()
t.emptyStatement = true
startPos := t.cursor
- startLine := t.line // Track the starting line number (1-based)
+ startLine := t.line // Track the starting line number (1-based)
+ startColumn := t.getColumn() // Track the starting column (1-based)
for {
switch {
case t.char(0) == eofRune:
@@ -232,6 +233,10 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
Text: s,
// BaseLine is 0-based line number, so subtract 1 from 1-based startLine
BaseLine: startLine - 1,
+ Start: &store.Position{
+ Line: int32(startLine),
+ Column: int32(startColumn),
+ },
// Consider this text:
// CREATE TABLE t(
// a int
@@ -244,7 +249,10 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
// but we want to get the line of last line of the SQL
// which means the line of ')'.
// So we need minus the aboveNonBlankLineDistance.
- End: &store.Position{Line: int32(t.line - t.aboveNonBlankLineDistance())},
+ End: &store.Position{
+ Line: int32(t.line - t.aboveNonBlankLineDistance()),
+ Column: int32(t.getLastContentColumn()),
+ },
Empty: t.emptyStatement,
Range: &store.Range{
Start: int32(t.getByteOffset(int(startPos))),
@@ -266,8 +274,15 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
Text: text,
// BaseLine is 0-based line number, so subtract 1 from 1-based startLine
BaseLine: startLine - 1,
- End: &store.Position{Line: int32(t.line)},
- Empty: t.emptyStatement,
+ Start: &store.Position{
+ Line: int32(startLine),
+ Column: int32(startColumn),
+ },
+ End: &store.Position{
+ Line: int32(t.line),
+ Column: int32(t.getColumn()),
+ },
+ Empty: t.emptyStatement,
Range: &store.Range{
Start: int32(t.getByteOffset(int(startPos))),
End: int32(t.getByteOffset(int(t.pos()))),
@@ -279,7 +294,8 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
return nil, err
}
startPos = t.pos()
- startLine = t.line // Update startLine for next statement
+ startLine = t.line // Update startLine for next statement
+ startColumn = t.getColumn() // Update startColumn for next statement
t.emptyStatement = true
// deal with the DELIMITER statement, see https://dev.mysql.com/doc/refman/8.0/en/stored-programs-defining.html
case t.equalWordCaseInsensitive(delimiterRuneList):
@@ -294,8 +310,15 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
Text: text,
// BaseLine is 0-based line number, so subtract 1 from 1-based startLine
BaseLine: startLine - 1,
- End: &store.Position{Line: int32(t.line)},
- Empty: false,
+ Start: &store.Position{
+ Line: int32(startLine),
+ Column: int32(startColumn),
+ },
+ End: &store.Position{
+ Line: int32(t.line),
+ Column: int32(t.getColumn()),
+ },
+ Empty: false,
Range: &store.Range{
Start: int32(t.getByteOffset(int(startPos))),
End: int32(t.getByteOffset(int(t.pos()))),
@@ -307,7 +330,8 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
return nil, err
}
startPos = t.pos()
- startLine = t.line // Update startLine for next statement
+ startLine = t.line // Update startLine for next statement
+ startColumn = t.getColumn() // Update startColumn for next statement
t.emptyStatement = true
case t.char(0) == '/' && t.char(1) == '*':
if err := t.scanComment(); err != nil {
@@ -719,13 +743,31 @@ func (t *Tokenizer) skipToNewLine() {
t.skip(1)
}
+// getColumn returns the 1-based column position of the current cursor.
+// Per the proto spec, the first character of a line is column 1.
func (t *Tokenizer) getColumn() int {
- for i := int(t.cursor) - 1; i >= 0; i-- {
+ return t.getColumnAt(t.cursor)
+}
+
+// getColumnAt returns the 1-based column position at a specific rune position.
+func (t *Tokenizer) getColumnAt(pos uint) int {
+ for i := int(pos) - 1; i >= 0; i-- {
if t.buffer[i] == '\n' {
- return int(t.cursor) - i - 1
+ return int(pos) - i // 1-based: first char after newline is column 1
+ }
+ }
+ return int(pos) + 1 // 1-based: first char of first line is column 1
+}
+
+// getLastContentColumn returns the 1-based column after the last non-blank character.
+// Used for EOF case where we need exclusive end position.
+func (t *Tokenizer) getLastContentColumn() int {
+ for i := int(t.cursor) - 1; i >= 0; i-- {
+ if !emptyRune(t.buffer[i]) && t.buffer[i] != eofRune {
+ return t.getColumnAt(uint(i)) + 1 // Exclusive: position after last content char
}
}
- return int(t.cursor)
+ return 1
}
func (t *Tokenizer) pos() uint {