Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser): add Start field to TiDB and Standard split implementations (#18628)
h3n4ldbb02d2119ec
diff --git a/backend/plugin/parser/standard/split.go b/backend/plugin/parser/standard/split.goindex 9da972dd5a..43a76644e4 100644--- a/backend/plugin/parser/standard/split.go+++ b/backend/plugin/parser/standard/split.go@@ -33,14 +33,27 @@ func SplitSQL(statement string) ([]base.Statement, error) {}endPos := startPos + len(sql)+ // Calculate line and column for Start position+ startLine, startColumn := calculateLineAndColumn(statement, startPos)+ // Calculate line and column for End position+ endLine, endColumn := calculateLineAndColumn(statement, endPos)+list = append(list, base.Statement{- Text: sql,+ Text: sql,+ BaseLine: startLine,+ Start: &storepb.Position{+ Line: int32(startLine + 1), // 1-based+ Column: int32(startColumn + 1), // 1-based per proto spec+ },+ End: &storepb.Position{+ Line: int32(endLine + 1), // 1-based+ Column: int32(endColumn + 1), // 1-based per proto spec+ },Range: &storepb.Range{Start: int32(startPos),End: int32(endPos),},- End: &storepb.Position{Line: 0, Column: 0},- Empty: false,+ Empty: isEmptySQL(sql),})byteOffset = endPosreturn nil@@ -48,6 +61,53 @@ func SplitSQL(statement string) ([]base.Statement, error) {return list, err}+// calculateLineAndColumn calculates the 0-based line number and 0-based column (character offset)+// for a given byte offset in the statement.+func calculateLineAndColumn(statement string, byteOffset int) (line, column int) {+ if byteOffset > len(statement) {+ byteOffset = len(statement)+ }+ // Range over string iterates over runes (code points), not bytes+ for _, r := range statement[:byteOffset] {+ if r == '\n' {+ line+++ column = 0+ } else {+ column+++ }+ }+ return line, column+}++// isEmptySQL checks if the SQL contains only whitespace and comments.+func isEmptySQL(sql string) bool {+ trimmed := strings.TrimSpace(sql)+ if trimmed == "" {+ return true+ }+ // Check if it's only comments+ if strings.HasPrefix(trimmed, "--") || strings.HasPrefix(trimmed, "/*") {+ // Simple heuristic: if after removing comment markers there's no SQL+ lines := strings.Split(trimmed, "\n")+ for _, line := range lines {+ line = strings.TrimSpace(line)+ if line == "" {+ continue+ }+ if strings.HasPrefix(line, "--") {+ continue+ }+ if strings.HasPrefix(line, "/*") && strings.HasSuffix(line, "*/") {+ continue+ }+ // Found non-comment content+ return false+ }+ return true+ }+ return false+}+// applyMultiStatements will apply the split statements from scanner.// This function only used for SQLite, snowflake.// For MySQL and PostgreSQL, use parser.SplitSQL.diff --git a/backend/plugin/parser/standard/split_test.go b/backend/plugin/parser/standard/split_test.goindex 33334b2bab..d17110ecd6 100644--- a/backend/plugin/parser/standard/split_test.go+++ b/backend/plugin/parser/standard/split_test.go@@ -8,6 +8,9 @@ import ("time""github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base")func generateOneMBInsert() string {@@ -67,3 +70,146 @@ func TestApplyMultiStatements(t *testing.T) {require.Equal(t, test.total, total)}}++func TestSplitSQL(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.Statement+ }{+ {+ name: "simple single statement",+ statement: "SELECT 1;",+ want: []base.Statement{+ {+ Text: "SELECT 1;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 10},+ Range: &storepb.Range{Start: 0, End: 9},+ Empty: false,+ },+ },+ },+ {+ name: "multi-line statement",+ statement: "SELECT\n 1;",+ want: []base.Statement{+ {+ Text: "SELECT\n 1;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 2, Column: 5},+ Range: &storepb.Range{Start: 0, End: 11},+ Empty: false,+ },+ },+ },+ {+ name: "multiple statements",+ statement: "SELECT 1;\nSELECT 2;",+ want: []base.Statement{+ {+ Text: "SELECT 1;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 10},+ Range: &storepb.Range{Start: 0, End: 9},+ Empty: false,+ },+ {+ Text: "SELECT 2;",+ BaseLine: 1,+ Start: &storepb.Position{Line: 2, Column: 1},+ End: &storepb.Position{Line: 2, Column: 10},+ Range: &storepb.Range{Start: 10, End: 19},+ Empty: false,+ },+ },+ },+ {+ name: "multi-byte characters - Chinese",+ statement: "SELECT '中文';",+ want: []base.Statement{+ {+ Text: "SELECT '中文';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ // Column is 1-based character offset: S(1) E(2) L(3) E(4) C(5) T(6) ' '(7) '(8) 中(9) 文(10) '(11) ;(12) = 12 chars, End is exclusive so 13+ End: &storepb.Position{Line: 1, Column: 13},+ Range: &storepb.Range{Start: 0, End: 16}, // byte length: 8 + 3 + 3 + 2 = 16+ Empty: false,+ },+ },+ },+ {+ name: "multi-byte characters - emoji",+ statement: "SELECT '🎉';",+ want: []base.Statement{+ {+ Text: "SELECT '🎉';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ // Column is 1-based character offset: S(1) E(2) L(3) E(4) C(5) T(6) ' '(7) '(8) 🎉(9) '(10) ;(11) = 11 chars, End is exclusive so 12+ End: &storepb.Position{Line: 1, Column: 12},+ Range: &storepb.Range{Start: 0, End: 14}, // byte length: 8 + 4 + 2 = 14+ Empty: false,+ },+ },+ },+ {+ name: "multi-byte on second line",+ statement: "SELECT\n '中文';",+ want: []base.Statement{+ {+ Text: "SELECT\n '中文';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ // Line 2 (1-based): ' '(1) ' '(2) '(3) 中(4) 文(5) '(6) ;(7) = 7 chars, End is exclusive so 8+ End: &storepb.Position{Line: 2, Column: 8},+ Range: &storepb.Range{Start: 0, End: 18}, // 7 + 3 + 3 + 3 + 2 = 18+ Empty: false,+ },+ },+ },+ {+ name: "multiple statements with multi-byte on separate lines",+ statement: "SELECT '中';\nSELECT '文';",+ want: []base.Statement{+ {+ Text: "SELECT '中';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ // S(1) E(2) L(3) E(4) C(5) T(6) ' '(7) '(8) 中(9) '(10) ;(11) = 11 chars, End is exclusive so 12+ End: &storepb.Position{Line: 1, Column: 12},+ Range: &storepb.Range{Start: 0, End: 13}, // 8 + 3 + 2 = 13+ Empty: false,+ },+ {+ Text: "SELECT '文';",+ BaseLine: 1,+ Start: &storepb.Position{Line: 2, Column: 1},+ End: &storepb.Position{Line: 2, Column: 12},+ Range: &storepb.Range{Start: 14, End: 27}, // starts after newline+ Empty: false,+ },+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ got, err := SplitSQL(tc.statement)+ require.NoError(t, err)+ require.Equal(t, len(tc.want), len(got), "number of statements mismatch")+ for i, want := range tc.want {+ require.Equal(t, want.Text, got[i].Text, "Text mismatch at index %d", i)+ require.Equal(t, want.BaseLine, got[i].BaseLine, "BaseLine mismatch at index %d", i)+ require.Equal(t, want.Start, got[i].Start, "Start mismatch at index %d", i)+ require.Equal(t, want.End, got[i].End, "End mismatch at index %d", i)+ require.Equal(t, want.Range, got[i].Range, "Range mismatch at index %d", i)+ require.Equal(t, want.Empty, got[i].Empty, "Empty mismatch at index %d", i)+ }+ })+ }+}diff --git a/backend/plugin/parser/tidb/split_test.go b/backend/plugin/parser/tidb/split_test.gonew file mode 100644index 0000000000..9581183fd4--- /dev/null+++ b/backend/plugin/parser/tidb/split_test.go@@ -0,0 +1,165 @@+package tidb++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+)++func TestSplitSQL(t *testing.T) {+ tests := []struct {+ name string+ statement string+ want []base.Statement+ }{+ {+ name: "simple single statement",+ statement: "SELECT 1;",+ want: []base.Statement{+ {+ Text: "SELECT 1;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 10}, // After semicolon (1-based exclusive)+ Range: &storepb.Range{Start: 0, End: 9},+ Empty: false,+ },+ },+ },+ {+ name: "multi-line statement",+ statement: "SELECT\n 1;",+ want: []base.Statement{+ {+ Text: "SELECT\n 1;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 2, Column: 5}, // After semicolon on line 2+ Range: &storepb.Range{Start: 0, End: 11},+ Empty: false,+ },+ },+ },+ {+ name: "multiple statements",+ statement: "SELECT 1;\nSELECT 2;",+ want: []base.Statement{+ {+ Text: "SELECT 1;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 10},+ Range: &storepb.Range{Start: 0, End: 9},+ Empty: false,+ },+ {+ Text: "SELECT 2;",+ BaseLine: 1,+ Start: &storepb.Position{Line: 2, Column: 1},+ End: &storepb.Position{Line: 2, Column: 10},+ Range: &storepb.Range{Start: 10, End: 19},+ Empty: false,+ },+ },+ },+ {+ name: "multi-byte characters - Chinese",+ statement: "SELECT '中文';",+ want: []base.Statement{+ {+ Text: "SELECT '中文';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ // Column is 1-based exclusive: S(1)..;(12), after = 13+ End: &storepb.Position{Line: 1, Column: 13},+ Range: &storepb.Range{Start: 0, End: 16}, // byte length: 8 + 3 + 3 + 2 = 16+ Empty: false,+ },+ },+ },+ {+ name: "multi-byte characters - emoji",+ statement: "SELECT '🎉';",+ want: []base.Statement{+ {+ Text: "SELECT '🎉';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ // Column is 1-based exclusive: S(1)..;(11), after = 12+ End: &storepb.Position{Line: 1, Column: 12},+ Range: &storepb.Range{Start: 0, End: 14}, // byte length: 8 + 4 + 2 = 14+ Empty: false,+ },+ },+ },+ {+ name: "multi-byte on second line",+ statement: "SELECT\n '中文';",+ want: []base.Statement{+ {+ Text: "SELECT\n '中文';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 2, Column: 8}, // After semicolon on line 2+ Range: &storepb.Range{Start: 0, End: 18}, // 7 + 3 + 3 + 3 + 2 = 18+ Empty: false,+ },+ },+ },+ {+ name: "multiple statements with multi-byte",+ statement: "SELECT '中';\nSELECT '文';",+ want: []base.Statement{+ {+ Text: "SELECT '中';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 12}, // After semicolon+ Range: &storepb.Range{Start: 0, End: 13}, // 8 + 3 + 2 = 13+ Empty: false,+ },+ {+ Text: "SELECT '文';",+ BaseLine: 1,+ Start: &storepb.Position{Line: 2, Column: 1},+ End: &storepb.Position{Line: 2, Column: 12},+ Range: &storepb.Range{Start: 14, End: 27},+ Empty: false,+ },+ },+ },+ {+ name: "statement with leading spaces and multi-byte",+ statement: " SELECT '中';",+ want: []base.Statement{+ {+ Text: "SELECT '中';",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 3}, // starts after 2 spaces (1-based: column 3)+ End: &storepb.Position{Line: 1, Column: 14}, // 2 leading spaces + 11 chars + 1 = 14+ Range: &storepb.Range{Start: 2, End: 15}, // 2 + 8 + 3 + 2 = 15+ Empty: false,+ },+ },+ },+ }++ for _, tc := range tests {+ t.Run(tc.name, func(t *testing.T) {+ got, err := SplitSQL(tc.statement)+ require.NoError(t, err)+ require.Equal(t, len(tc.want), len(got), "number of statements mismatch")+ for i, want := range tc.want {+ require.Equal(t, want.Text, got[i].Text, "Text mismatch at index %d", i)+ require.Equal(t, want.BaseLine, got[i].BaseLine, "BaseLine mismatch at index %d", i)+ require.Equal(t, want.Start, got[i].Start, "Start mismatch at index %d", i)+ require.Equal(t, want.End, got[i].End, "End mismatch at index %d", i)+ require.Equal(t, want.Range, got[i].Range, "Range mismatch at index %d", i)+ require.Equal(t, want.Empty, got[i].Empty, "Empty mismatch at index %d", i)+ }+ })+ }+}diff --git a/backend/plugin/parser/tokenizer/tokenizer.go b/backend/plugin/parser/tokenizer/tokenizer.goindex 434a445157..125896b375 100644--- a/backend/plugin/parser/tokenizer/tokenizer.go+++ b/backend/plugin/parser/tokenizer/tokenizer.go@@ -221,7 +221,8 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {t.skipBlank()t.emptyStatement = truestartPos := t.cursor- startLine := t.line // Track the starting line number (1-based)+ startLine := t.line // Track the starting line number (1-based)+ startColumn := t.getColumn() // Track the starting column (1-based)for {switch {case t.char(0) == eofRune:@@ -232,6 +233,10 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {Text: s,// BaseLine is 0-based line number, so subtract 1 from 1-based startLineBaseLine: startLine - 1,+ Start: &store.Position{+ Line: int32(startLine),+ Column: int32(startColumn),+ },// Consider this text:// CREATE TABLE t(// a int@@ -244,7 +249,10 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {// but we want to get the line of last line of the SQL// which means the line of ')'.// So we need minus the aboveNonBlankLineDistance.- End: &store.Position{Line: int32(t.line - t.aboveNonBlankLineDistance())},+ End: &store.Position{+ Line: int32(t.line - t.aboveNonBlankLineDistance()),+ Column: int32(t.getLastContentColumn()),+ },Empty: t.emptyStatement,Range: &store.Range{Start: int32(t.getByteOffset(int(startPos))),@@ -266,8 +274,15 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {Text: text,// BaseLine is 0-based line number, so subtract 1 from 1-based startLineBaseLine: startLine - 1,- End: &store.Position{Line: int32(t.line)},- Empty: t.emptyStatement,+ Start: &store.Position{+ Line: int32(startLine),+ Column: int32(startColumn),+ },+ End: &store.Position{+ Line: int32(t.line),+ Column: int32(t.getColumn()),+ },+ Empty: t.emptyStatement,Range: &store.Range{Start: int32(t.getByteOffset(int(startPos))),End: int32(t.getByteOffset(int(t.pos()))),@@ -279,7 +294,8 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {return nil, err}startPos = t.pos()- startLine = t.line // Update startLine for next statement+ startLine = t.line // Update startLine for next statement+ startColumn = t.getColumn() // Update startColumn for next statementt.emptyStatement = true// deal with the DELIMITER statement, see https://dev.mysql.com/doc/refman/8.0/en/stored-programs-defining.htmlcase t.equalWordCaseInsensitive(delimiterRuneList):@@ -294,8 +310,15 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {Text: text,// BaseLine is 0-based line number, so subtract 1 from 1-based startLineBaseLine: startLine - 1,- End: &store.Position{Line: int32(t.line)},- Empty: false,+ Start: &store.Position{+ Line: int32(startLine),+ Column: int32(startColumn),+ },+ End: &store.Position{+ Line: int32(t.line),+ Column: int32(t.getColumn()),+ },+ Empty: false,Range: &store.Range{Start: int32(t.getByteOffset(int(startPos))),End: int32(t.getByteOffset(int(t.pos()))),@@ -307,7 +330,8 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {return nil, err}startPos = t.pos()- startLine = t.line // Update startLine for next statement+ startLine = t.line // Update startLine for next statement+ startColumn = t.getColumn() // Update startColumn for next statementt.emptyStatement = truecase t.char(0) == '/' && t.char(1) == '*':if err := t.scanComment(); err != nil {@@ -719,13 +743,31 @@ func (t *Tokenizer) skipToNewLine() {t.skip(1)}+// getColumn returns the 1-based column position of the current cursor.+// Per the proto spec, the first character of a line is column 1.func (t *Tokenizer) getColumn() int {- for i := int(t.cursor) - 1; i >= 0; i-- {+ return t.getColumnAt(t.cursor)+}++// getColumnAt returns the 1-based column position at a specific rune position.+func (t *Tokenizer) getColumnAt(pos uint) int {+ for i := int(pos) - 1; i >= 0; i-- {if t.buffer[i] == '\n' {- return int(t.cursor) - i - 1+ return int(pos) - i // 1-based: first char after newline is column 1+ }+ }+ return int(pos) + 1 // 1-based: first char of first line is column 1+}++// getLastContentColumn returns the 1-based column after the last non-blank character.+// Used for EOF case where we need exclusive end position.+func (t *Tokenizer) getLastContentColumn() int {+ for i := int(t.cursor) - 1; i >= 0; i-- {+ if !emptyRune(t.buffer[i]) && t.buffer[i] != eofRune {+ return t.getColumnAt(uint(i)) + 1 // Exclusive: position after last content char}}- return int(t.cursor)+ return 1}func (t *Tokenizer) pos() uint {