Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(parser): standardize position fields in parser split infrastructure (#18645)

h3n4l4293e3384268
diff --git a/backend/plugin/parser/cosmosdb/split_test.go b/backend/plugin/parser/cosmosdb/split_test.go
index 0d748a2fd8..e2fc3326bc 100644
--- a/backend/plugin/parser/cosmosdb/split_test.go
+++ b/backend/plugin/parser/cosmosdb/split_test.go
@@ -4,36 +4,62 @@ import (
"testing"
"github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
)
func TestSplitSQL(t *testing.T) {
tests := []struct {
name string
statement string
- wantCount int
- wantEmpty bool
+ want *base.Statement // nil means no statements expected
}{
{
name: "Single SELECT statement",
statement: "SELECT * FROM c",
- wantCount: 1,
- wantEmpty: false,
+ want: &base.Statement{
+ Text: "SELECT * FROM c",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 16}, // 1-based exclusive (after 'c')
+ Range: &storepb.Range{Start: 0, End: 15},
+ Empty: false,
+ },
},
{
name: "SELECT with WHERE",
statement: "SELECT c.id FROM users c WHERE c.active = true",
- wantCount: 1,
- wantEmpty: false,
+ want: &base.Statement{
+ Text: "SELECT c.id FROM users c WHERE c.active = true",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 47}, // 1-based exclusive (after 'e')
+ Range: &storepb.Range{Start: 0, End: 46},
+ Empty: false,
+ },
+ },
+ {
+ name: "Multi-line SELECT",
+ statement: "SELECT\n * FROM c",
+ want: &base.Statement{
+ Text: "SELECT\n * FROM c",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 11}, // 1-based exclusive (after 'c')
+ Range: &storepb.Range{Start: 0, End: 17},
+ Empty: false,
+ },
},
{
name: "Empty string",
statement: "",
- wantCount: 0,
+ want: nil,
},
{
name: "Only whitespace",
statement: " \n \t ",
- wantCount: 0,
+ want: nil,
},
}
@@ -41,15 +67,30 @@ func TestSplitSQL(t *testing.T) {
t.Run(tt.name, func(t *testing.T) {
list, err := SplitSQL(tt.statement)
require.NoError(t, err)
- require.Equal(t, tt.wantCount, len(list))
-
- if tt.wantCount > 0 {
- require.Equal(t, tt.statement, list[0].Text)
- require.Equal(t, 0, list[0].BaseLine)
- require.Equal(t, tt.wantEmpty, list[0].Empty)
- require.NotNil(t, list[0].Start)
- require.NotNil(t, list[0].End)
+
+ if tt.want == nil {
+ require.Empty(t, list)
+ return
}
+
+ require.Len(t, list, 1)
+ got := list[0]
+
+ require.Equal(t, tt.want.Text, got.Text, "Text mismatch")
+ require.Equal(t, tt.want.BaseLine, got.BaseLine, "BaseLine mismatch")
+ require.Equal(t, tt.want.Empty, got.Empty, "Empty mismatch")
+
+ require.NotNil(t, got.Start, "Start should not be nil")
+ require.Equal(t, tt.want.Start.Line, got.Start.Line, "Start.Line mismatch")
+ require.Equal(t, tt.want.Start.Column, got.Start.Column, "Start.Column mismatch")
+
+ require.NotNil(t, got.End, "End should not be nil")
+ require.Equal(t, tt.want.End.Line, got.End.Line, "End.Line mismatch")
+ require.Equal(t, tt.want.End.Column, got.End.Column, "End.Column mismatch")
+
+ require.NotNil(t, got.Range, "Range should not be nil")
+ require.Equal(t, tt.want.Range.Start, got.Range.Start, "Range.Start mismatch")
+ require.Equal(t, tt.want.Range.End, got.Range.End, "Range.End mismatch")
})
}
}
diff --git a/backend/plugin/parser/doris/split_test.go b/backend/plugin/parser/doris/split_test.go
index 1b3e84f5f4..0a7750cc9b 100644
--- a/backend/plugin/parser/doris/split_test.go
+++ b/backend/plugin/parser/doris/split_test.go
@@ -4,28 +4,106 @@ import (
"testing"
"github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
)
func TestSplitDorisSQLStatements(t *testing.T) {
tests := []struct {
+ name string
statement string
- expected []string
+ want []base.Statement
}{
{
- statement: "SELECT * FROM t1 WHERE c1 = 1; SELECT * FROM t2;",
- expected: []string{
- "SELECT * FROM t1 WHERE c1 = 1;",
- " SELECT * FROM t2;",
+ name: "simple single statement",
+ statement: "SELECT 1;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 9},
+ Range: &storepb.Range{Start: 0, End: 9},
+ },
+ },
+ },
+ {
+ name: "multiple statements on one line",
+ statement: "SELECT 1; SELECT 2;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 9},
+ Range: &storepb.Range{Start: 0, End: 9},
+ },
+ {
+ Text: " SELECT 2;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 11},
+ End: &storepb.Position{Line: 1, Column: 19},
+ Range: &storepb.Range{Start: 9, End: 19},
+ },
+ },
+ },
+ {
+ name: "multi-line statement",
+ statement: "SELECT\n 1;",
+ want: []base.Statement{
+ {
+ Text: "SELECT\n 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 4},
+ Range: &storepb.Range{Start: 0, End: 11},
+ },
+ },
+ },
+ {
+ name: "multiple statements on multiple lines",
+ statement: "SELECT 1;\nSELECT 2;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 9},
+ Range: &storepb.Range{Start: 0, End: 9},
+ },
+ {
+ Text: "\nSELECT 2;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 2, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 9},
+ Range: &storepb.Range{Start: 9, End: 19},
+ },
},
},
}
- for _, test := range tests {
- list, err := SplitSQL(test.statement)
- require.NoError(t, err)
- require.Equal(t, len(test.expected), len(list))
- for i, statement := range list {
- require.Equal(t, test.expected[i], statement.Text)
- }
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ list, err := SplitSQL(tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, len(tc.want), len(list), "statement count mismatch")
+
+ for i, want := range tc.want {
+ got := list[i]
+ require.Equal(t, want.Text, got.Text, "Text mismatch at index %d", i)
+ require.Equal(t, want.BaseLine, got.BaseLine, "BaseLine mismatch at index %d", i)
+ require.NotNil(t, got.Start, "Start should not be nil at index %d", i)
+ require.Equal(t, want.Start.Line, got.Start.Line, "Start.Line mismatch at index %d", i)
+ require.Equal(t, want.Start.Column, got.Start.Column, "Start.Column mismatch at index %d", i)
+ require.NotNil(t, got.End, "End should not be nil at index %d", i)
+ require.Equal(t, want.End.Line, got.End.Line, "End.Line mismatch at index %d", i)
+ require.Equal(t, want.End.Column, got.End.Column, "End.Column mismatch at index %d", i)
+ if want.Range != nil {
+ require.Equal(t, want.Range.Start, got.Range.Start, "Range.Start mismatch at index %d", i)
+ require.Equal(t, want.Range.End, got.Range.End, "Range.End mismatch at index %d", i)
+ }
+ }
+ })
}
}
diff --git a/backend/plugin/parser/mysql/completion.go b/backend/plugin/parser/mysql/completion.go
index a7e80998df..f1d3ae9252 100644
--- a/backend/plugin/parser/mysql/completion.go
+++ b/backend/plugin/parser/mysql/completion.go
@@ -1235,7 +1235,7 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
start := 0
for i, sql := range list {
- // sql.End uses 1-based line and column
+ // Both caretLine and End.Line are 1-based
sqlEndLine := int(sql.End.GetLine())
sqlEndColumn := int(sql.End.GetColumn())
if sqlEndLine > caretLine || (sqlEndLine == caretLine && sqlEndColumn >= caretOffset) {
diff --git a/backend/plugin/parser/partiql/completion_test.go b/backend/plugin/parser/partiql/completion_test.go
index 52443ed33e..50a87e8c6f 100644
--- a/backend/plugin/parser/partiql/completion_test.go
+++ b/backend/plugin/parser/partiql/completion_test.go
@@ -74,7 +74,7 @@ func TestCompletion(t *testing.T) {
if record {
tests[i].Want = filteredResult
} else {
- a.Equalf(t.Want, filteredResult, t.Input, "Case %02d: %s", i, t.Description)
+ a.Equalf(t.Want, filteredResult, "Case %02d: %s", i, t.Input)
}
}
diff --git a/backend/plugin/parser/pg/completion.go b/backend/plugin/parser/pg/completion.go
index f55da69b39..838097a3ae 100644
--- a/backend/plugin/parser/pg/completion.go
+++ b/backend/plugin/parser/pg/completion.go
@@ -1278,7 +1278,7 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
start := 0
for i, sql := range list {
- // sql.End uses 1-based line and column
+ // Both caretLine and End.Line are 1-based
sqlEndLine := int(sql.End.GetLine())
sqlEndColumn := int(sql.End.GetColumn())
if sqlEndLine > caretLine || (sqlEndLine == caretLine && sqlEndColumn >= caretOffset) {
diff --git a/backend/plugin/parser/plsql/completion.go b/backend/plugin/parser/plsql/completion.go
index 504b03c7bf..a74661dd0a 100644
--- a/backend/plugin/parser/plsql/completion.go
+++ b/backend/plugin/parser/plsql/completion.go
@@ -1156,7 +1156,8 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
start := 0
for i, sql := range list {
- sqlEndLine := int(sql.End.GetLine())
+ // End.Line is 1-based per proto spec, convert to 0-based for comparison with caretLine
+ sqlEndLine := int(sql.End.GetLine()) - 1
sqlEndColumn := int(sql.End.GetColumn())
if sqlEndLine > caretLine || (sqlEndLine == caretLine && sqlEndColumn >= caretOffset) {
start = i
@@ -1164,7 +1165,8 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
// The caret is in the first SQL statement, so we don't need to skip any SQL statements.
break
}
- previousSQLEndLine := int(list[i-1].End.GetLine())
+ // End.Line is 1-based per proto spec, convert to 0-based
+ previousSQLEndLine := int(list[i-1].End.GetLine()) - 1
previousSQLEndColumn := int(list[i-1].End.GetColumn())
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
diff --git a/backend/plugin/parser/redshift/completion.go b/backend/plugin/parser/redshift/completion.go
index ac15058f47..e92097e1f1 100644
--- a/backend/plugin/parser/redshift/completion.go
+++ b/backend/plugin/parser/redshift/completion.go
@@ -1272,7 +1272,8 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
start := 0
for i, sql := range list {
- sqlEndLine := int(sql.End.GetLine())
+ // End.Line is 1-based per proto spec, convert to 0-based for comparison with caretLine
+ sqlEndLine := int(sql.End.GetLine()) - 1
sqlEndColumn := int(sql.End.GetColumn())
if sqlEndLine > caretLine || (sqlEndLine == caretLine && sqlEndColumn >= caretOffset) {
start = i
@@ -1280,7 +1281,8 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
// The caret is in the first SQL statement, so we don't need to skip any SQL statements.
break
}
- previousSQLEndLine := int(list[i-1].End.GetLine())
+ // End.Line is 1-based per proto spec, convert to 0-based
+ previousSQLEndLine := int(list[i-1].End.GetLine()) - 1
previousSQLEndColumn := int(list[i-1].End.GetColumn())
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
diff --git a/backend/plugin/parser/redshift/split_test.go b/backend/plugin/parser/redshift/split_test.go
new file mode 100644
index 0000000000..f2c7e68c71
--- /dev/null
+++ b/backend/plugin/parser/redshift/split_test.go
@@ -0,0 +1,112 @@
+package redshift
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+)
+
+func TestSplitSQL(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ want []base.Statement
+ }{
+ {
+ name: "simple single statement",
+ statement: "SELECT 1;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 9},
+ Range: &storepb.Range{Start: 0, End: 9},
+ },
+ },
+ },
+ {
+ name: "multiple statements",
+ statement: "SELECT 1; SELECT 2;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 9},
+ Range: &storepb.Range{Start: 0, End: 9},
+ },
+ {
+ Text: " SELECT 2;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 11},
+ End: &storepb.Position{Line: 1, Column: 19},
+ Range: &storepb.Range{Start: 9, End: 19},
+ },
+ },
+ },
+ {
+ name: "multi-line statement",
+ statement: "SELECT\n 1;",
+ want: []base.Statement{
+ {
+ Text: "SELECT\n 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 4},
+ Range: &storepb.Range{Start: 0, End: 11},
+ },
+ },
+ },
+ {
+ name: "multiple statements on multiple lines",
+ statement: "SELECT 1;\nSELECT 2;",
+ want: []base.Statement{
+ {
+ Text: "SELECT 1;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 9},
+ Range: &storepb.Range{Start: 0, End: 9},
+ },
+ {
+ Text: "\nSELECT 2;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 2, Column: 1},
+ End: &storepb.Position{Line: 2, Column: 9},
+ Range: &storepb.Range{Start: 9, End: 19},
+ },
+ },
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ list, err := SplitSQL(tc.statement)
+ require.NoError(t, err)
+ require.Equal(t, len(tc.want), len(list), "statement count mismatch")
+
+ for i, want := range tc.want {
+ got := list[i]
+ require.Equal(t, want.Text, got.Text, "Text mismatch at index %d", i)
+ require.Equal(t, want.BaseLine, got.BaseLine, "BaseLine mismatch at index %d", i)
+
+ require.NotNil(t, got.Start, "Start should not be nil at index %d", i)
+ require.Equal(t, want.Start.Line, got.Start.Line, "Start.Line mismatch at index %d", i)
+ require.Equal(t, want.Start.Column, got.Start.Column, "Start.Column mismatch at index %d", i)
+
+ require.NotNil(t, got.End, "End should not be nil at index %d", i)
+ require.Equal(t, want.End.Line, got.End.Line, "End.Line mismatch at index %d", i)
+ require.Equal(t, want.End.Column, got.End.Column, "End.Column mismatch at index %d", i)
+
+ if want.Range != nil {
+ require.Equal(t, want.Range.Start, got.Range.Start, "Range.Start mismatch at index %d", i)
+ require.Equal(t, want.Range.End, got.Range.End, "Range.End mismatch at index %d", i)
+ }
+ }
+ })
+ }
+}
diff --git a/backend/plugin/parser/tokenizer/tokenizer.go b/backend/plugin/parser/tokenizer/tokenizer.go
index 125896b375..a753deac3b 100644
--- a/backend/plugin/parser/tokenizer/tokenizer.go
+++ b/backend/plugin/parser/tokenizer/tokenizer.go
@@ -416,15 +416,16 @@ func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {
text := t.getString(startPos, t.pos()-startPos)
if t.f == nil {
res = append(res, base.Statement{
- Text: text,
- End: &store.Position{
- Line: int32(t.line - 1), // Convert to 0-based.
- },
- // TODO(zp/position): fix column, use bytes instead of rune.
+ Text: text,
+ BaseLine: firstStatementLine - 1, // 0-based line of first token
Start: &store.Position{
- Line: int32(firstStatementLine - 1), // Convert to 0-based.
+ Line: int32(firstStatementLine), // 1-based per proto spec
Column: int32(firstStatementColumn),
},
+ End: &store.Position{
+ Line: int32(t.line), // 1-based per proto spec
+ Column: int32(t.getColumn()), // 1-based exclusive (after semicolon)
+ },
Empty: t.emptyStatement,
Range: &store.Range{
Start: int32(t.getByteOffset(int(startPos))),
@@ -444,26 +445,20 @@ func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {
s := t.getString(startPos, t.pos())
if !emptyString(s) {
if t.f == nil {
+ // For EOF case, we need to find the position of the last content character.
+ // The cursor might be past all content (on blank lines or at EOF).
+ endLine := t.line - t.aboveNonBlankLineDistance()
res = append(res, base.Statement{
- Text: s,
- // Consider this text:
- // CREATE TABLE t(
- // a int
- // )
- //
- // EOF line
- //
- // Our current location is the EOF line.
- // The line t.line is the line of ')',
- // but we want to get the line of last line of the SQL
- // which means the line of ')'.
- // So we need minus the aboveNonBlankLineDistance.
- End: &store.Position{Line: int32(t.line - t.aboveNonBlankLineDistance() - 1)},
- // TODO(zp/position): fix column, use bytes instead of rune.
+ Text: s,
+ BaseLine: firstStatementLine - 1, // 0-based line of first token
Start: &store.Position{
- Line: int32(firstStatementLine - 1), // Convert to 0-based.
+ Line: int32(firstStatementLine), // 1-based per proto spec
Column: int32(firstStatementColumn),
},
+ End: &store.Position{
+ Line: int32(endLine), // 1-based per proto spec
+ Column: int32(t.getLastContentColumn()), // 1-based exclusive
+ },
Empty: t.emptyStatement,
Range: &store.Range{
Start: int32(t.getByteOffset(int(startPos))),
diff --git a/backend/plugin/parser/trino/completion.go b/backend/plugin/parser/trino/completion.go
index 47e6064867..8ac4831a10 100644
--- a/backend/plugin/parser/trino/completion.go
+++ b/backend/plugin/parser/trino/completion.go
@@ -1130,7 +1130,8 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
start := 0
for i, sql := range list {
- sqlEndLine := int(sql.End.GetLine())
+ // End.Line is 1-based per proto spec, convert to 0-based for comparison with caretLine
+ sqlEndLine := int(sql.End.GetLine()) - 1
sqlEndColumn := int(sql.End.GetColumn())
if sqlEndLine > caretLine || (sqlEndLine == caretLine && sqlEndColumn >= caretOffset) {
start = i
@@ -1138,7 +1139,8 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
// The caret is in the first SQL statement, so we don't need to skip any SQL statements.
break
}
- previousSQLEndLine := int(list[i-1].End.GetLine())
+ // End.Line is 1-based per proto spec, convert to 0-based
+ previousSQLEndLine := int(list[i-1].End.GetLine()) - 1
previousSQLEndColumn := int(list[i-1].End.GetColumn())
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
diff --git a/backend/plugin/parser/trino/split.go b/backend/plugin/parser/trino/split.go
index 5d8e9beb4c..182a79d3d8 100644
--- a/backend/plugin/parser/trino/split.go
+++ b/backend/plugin/parser/trino/split.go
@@ -1,8 +1,6 @@
package trino
import (
- "strings"
-
"github.com/antlr4-go/antlr/v4"
trinoparser "github.com/bytebase/parser/trino"
@@ -28,76 +26,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {
func splitByTokenizer(statement string) ([]base.Statement, error) {
t := tokenizer.NewTokenizer(statement)
- list, err := t.SplitStandardMultiSQL()
- if err != nil {
- return nil, err
- }
-
- // The tokenizer doesn't provide accurate position information.
- // We need to manually calculate positions based on the text.
- lines := strings.Split(statement, "\n")
- lineStarts := make([]int, len(lines))
- pos := 0
- for i, line := range lines {
- lineStarts[i] = pos
- pos += len(line) + 1 // +1 for newline
- }
-
- currentPos := 0
- for i := range list {
- if list[i].Empty {
- continue
- }
-
- // Find the start position
- startIdx := strings.Index(statement[currentPos:], list[i].Text)
- if startIdx == -1 {
- continue
- }
- startIdx += currentPos
-
- // Calculate line and column for start
- startLine, startCol := 0, 0
- for j, lineStart := range lineStarts {
- if startIdx < lineStart {
- break
- }
- startLine = j
- startCol = startIdx - lineStart
- }
-
- // Calculate end position
- endIdx := startIdx + len(list[i].Text)
- endLine, endCol := 0, 0
- for j, lineStart := range lineStarts {
- if endIdx <= lineStart {
- break
- }
- endLine = j
- endCol = endIdx - lineStart
- }
- // If we're exactly at the start of a new line, we're at the end of the previous line
- if endCol == 0 && endLine > 0 {
- endLine--
- if endLine < len(lines) {
- endCol = len(lines[endLine])
- }
- }
-
- list[i].Start = &storepb.Position{
- Line: int32(startLine),
- Column: int32(startCol),
- }
- list[i].End = &storepb.Position{
- Line: int32(endLine),
- Column: int32(endCol),
- }
- list[i].BaseLine = startLine
-
- currentPos = endIdx
- }
-
- return list, nil
+ return t.SplitStandardMultiSQL()
}
func splitByParser(statement string) ([]base.Statement, error) {
@@ -133,7 +62,6 @@ func splitByParser(statement string) ([]base.Statement, error) {
var result []base.Statement
tokens := stream.GetAllTokens()
- byteOffset := 0
// Walk through all statements
for _, stmts := range tree.AllStatements() {
if stmts == nil {
@@ -183,31 +111,32 @@ func splitByParser(statement string) ([]base.Statement, error) {
// Get the text including any trailing semicolon
text := stream.GetTextFromInterval(antlr.NewInterval(startIdx, finalEndIdx))
- stmtByteLength := len(text)
- // Calculate proper end position
+ // Calculate proper end position (1-based exclusive per proto spec)
endToken := tokens[finalEndIdx]
- endLine := endToken.GetLine() - 1
- endColumn := endToken.GetColumn() + len(endToken.GetText())
+
+ // Use actual token positions for Range instead of cumulative offset
+ // GetStart() returns byte offset of first character, GetStop() returns byte offset of last character
+ rangeStart := firstDefaultToken.GetStart()
+ rangeEnd := endToken.GetStop() + 1 // exclusive end
result = append(result, base.Statement{
Text: text,
BaseLine: firstToken.GetLine() - 1,
Range: &storepb.Range{
- Start: int32(byteOffset),
- End: int32(byteOffset + stmtByteLength),
+ Start: int32(rangeStart),
+ End: int32(rangeEnd),
},
Start: &storepb.Position{
- Line: int32(firstDefaultToken.GetLine() - 1),
- Column: int32(firstDefaultToken.GetColumn()),
+ Line: int32(firstDefaultToken.GetLine()), // 1-based (ANTLR line is already 1-based)
+ Column: int32(firstDefaultToken.GetColumn() + 1), // 1-based (ANTLR column is 0-based)
},
End: &storepb.Position{
- Line: int32(endLine),
- Column: int32(endColumn),
+ Line: int32(endToken.GetLine()), // 1-based
+ Column: int32(endToken.GetColumn() + len(endToken.GetText()) + 1), // 1-based exclusive
},
Empty: false,
})
- byteOffset += stmtByteLength
}
return result, nil
diff --git a/backend/plugin/parser/trino/split_test.go b/backend/plugin/parser/trino/split_test.go
index 040af12f66..51ca96890f 100644
--- a/backend/plugin/parser/trino/split_test.go
+++ b/backend/plugin/parser/trino/split_test.go
@@ -28,14 +28,14 @@ func TestTrinoSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: "SELECT * FROM users;",
- Start: &storepb.Position{Line: 0, Column: 0},
- End: &storepb.Position{Line: 0, Column: 20},
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 21},
},
{
Text: "SELECT * FROM orders;",
BaseLine: 0,
- Start: &storepb.Position{Line: 0, Column: 21},
- End: &storepb.Position{Line: 0, Column: 42},
+ Start: &storepb.Position{Line: 1, Column: 22},
+ End: &storepb.Position{Line: 1, Column: 43},
},
},
},
@@ -43,32 +43,32 @@ func TestTrinoSplitMultiSQL(t *testing.T) {
{
statement: `
-- This is a comment
- SELECT
- id,
- name
- FROM users
+ SELECT
+ id,
+ name
+ FROM users
WHERE status = 'active';
-
+
/* This is a multi-line
comment */
SELECT * FROM orders;`,
want: resData{
res: []base.Statement{
{
- Text: `SELECT
- id,
- name
- FROM users
+ Text: `SELECT
+ id,
+ name
+ FROM users
WHERE status = 'active';`,
BaseLine: 2,
- Start: &storepb.Position{Line: 2, Column: 4},
- End: &storepb.Position{Line: 6, Column: 28},
+ Start: &storepb.Position{Line: 3, Column: 5},
+ End: &storepb.Position{Line: 7, Column: 29},
},
{
Text: `SELECT * FROM orders;`,
BaseLine: 10,
- Start: &storepb.Position{Line: 10, Column: 4},
- End: &storepb.Position{Line: 10, Column: 25},
+ Start: &storepb.Position{Line: 11, Column: 5},
+ End: &storepb.Position{Line: 11, Column: 26},
},
},
},
@@ -77,10 +77,10 @@ func TestTrinoSplitMultiSQL(t *testing.T) {
statement: `WITH orders_cte AS (
SELECT * FROM orders
)
- SELECT u.id, u.name, o.order_id
+ SELECT u.id, u.name, o.order_id
FROM users u
JOIN orders_cte o ON u.id = o.user_id;
-
+
SELECT * FROM products;`,
want: resData{
res: []base.Statement{
@@ -88,36 +88,38 @@ func TestTrinoSplitMultiSQL(t *testing.T) {
Text: `WITH orders_cte AS (
SELECT * FROM orders
)
- SELECT u.id, u.name, o.order_id
+ SELECT u.id, u.name, o.order_id
FROM users u
JOIN orders_cte o ON u.id = o.user_id;`,
- Start: &storepb.Position{Line: 0, Column: 0},
- End: &storepb.Position{Line: 5, Column: 42},
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 6, Column: 43},
},
{
Text: `SELECT * FROM products;`,
BaseLine: 7,
- Start: &storepb.Position{Line: 7, Column: 4},
- End: &storepb.Position{Line: 7, Column: 27},
+ Start: &storepb.Position{Line: 8, Column: 5},
+ End: &storepb.Position{Line: 8, Column: 28},
},
},
},
},
{
+ // Note: This test falls back to tokenizer path (parser fails with Chinese chars)
+ // Tokenizer uses character-based column offsets
statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",
want: resData{
res: []base.Statement{
{
Text: "SELECT * FROM 表名;",
- Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)
- Start: &storepb.Position{Line: 0, Column: 0},
- End: &storepb.Position{Line: 0, Column: 17},
+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 18}, // Character-based: 17 chars + 1
},
{
Text: "INSERT INTO 表 VALUES (1);",
- Range: &storepb.Range{Start: 21, End: 48}, // Byte offset 21-48 (not 17-42)
- Start: &storepb.Position{Line: 0, Column: 18},
- End: &storepb.Position{Line: 0, Column: 42},
+ Range: &storepb.Range{Start: 22, End: 49}, // Byte offset 22-49
+ Start: &storepb.Position{Line: 1, Column: 19},
+ End: &storepb.Position{Line: 1, Column: 44}, // Character-based exclusive
},
},
},
@@ -132,16 +134,78 @@ func TestTrinoSplitMultiSQL(t *testing.T) {
errStr = err.Error()
}
- // Skip position checks for simplicity - just check the text content
- for i, sql := range res {
- if i < len(test.want.res) {
- require.Equal(t, test.want.res[i].Text, sql.Text,
- "Expected statement text to match for statement %d", i)
- }
- }
require.Equal(t, len(test.want.res), len(res),
"Expected %d statements, got %d", len(test.want.res), len(res))
require.Equal(t, test.want.err, errStr,
"Expected error '%s', got '%s'", test.want.err, errStr)
+
+ // Check each statement's fields
+ for i, want := range test.want.res {
+ got := res[i]
+ require.Equal(t, want.Text, got.Text, "Text mismatch at index %d", i)
+ require.Equal(t, want.Start, got.Start, "Start mismatch at index %d", i)
+ require.Equal(t, want.End, got.End, "End mismatch at index %d", i)
+ if want.Range != nil {
+ require.Equal(t, want.Range, got.Range, "Range mismatch at index %d", i)
+ }
+ }
+ }
+}
+
+// TestSplitByParserAndTokenizerConsistency verifies that both splitByParser and
+// splitByTokenizer return consistent results for the same valid SQL input.
+func TestSplitByParserAndTokenizerConsistency(t *testing.T) {
+ tests := []struct {
+ name string
+ statement string
+ }{
+ {
+ name: "simple single statement",
+ statement: "SELECT 1;",
+ },
+ {
+ name: "multiple statements on one line",
+ statement: "SELECT 1; SELECT 2;",
+ },
+ {
+ name: "multi-line statement",
+ statement: "SELECT\n 1;",
+ },
+ {
+ name: "multiple multi-line statements",
+ statement: "SELECT\n 1;\nSELECT\n 2;",
+ },
+ {
+ name: "statement with trailing whitespace",
+ statement: "SELECT 1; \n ",
+ },
+ }
+
+ for _, tc := range tests {
+ t.Run(tc.name, func(t *testing.T) {
+ parserRes, parserErr := splitByParser(tc.statement)
+ tokenizerRes, tokenizerErr := splitByTokenizer(tc.statement)
+
+ // Both should succeed for valid SQL
+ require.NoError(t, parserErr, "parser should not error")
+ require.NoError(t, tokenizerErr, "tokenizer should not error")
+
+ // Should return same number of statements
+ require.Equal(t, len(parserRes), len(tokenizerRes),
+ "statement count mismatch: parser=%d, tokenizer=%d", len(parserRes), len(tokenizerRes))
+
+ // Compare each statement's fields
+ for i := range parserRes {
+ p := parserRes[i]
+ tok := tokenizerRes[i]
+
+ require.Equal(t, p.Text, tok.Text, "Text mismatch at index %d", i)
+ require.Equal(t, p.BaseLine, tok.BaseLine, "BaseLine mismatch at index %d", i)
+ require.Equal(t, p.Start, tok.Start, "Start mismatch at index %d", i)
+ require.Equal(t, p.End, tok.End, "End mismatch at index %d", i)
+ require.Equal(t, p.Range, tok.Range, "Range mismatch at index %d", i)
+ require.Equal(t, p.Empty, tok.Empty, "Empty mismatch at index %d", i)
+ }
+ })
}
}
diff --git a/backend/plugin/parser/tsql/completion.go b/backend/plugin/parser/tsql/completion.go
index 7606415b66..d10de7bfff 100644
--- a/backend/plugin/parser/tsql/completion.go
+++ b/backend/plugin/parser/tsql/completion.go
@@ -1305,7 +1305,8 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
start := 0
for i, sql := range list {
- sqlEndLine := int(sql.End.GetLine())
+ // End.Line is 1-based per proto spec, convert to 0-based for comparison with caretLine
+ sqlEndLine := int(sql.End.GetLine()) - 1
sqlEndColumn := int(sql.End.GetColumn())
if sqlEndLine > caretLine || (sqlEndLine == caretLine && sqlEndColumn >= caretOffset) {
start = i
@@ -1313,7 +1314,8 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
// The caret is in the first SQL statement, so we don't need to skip any SQL statements.
break
}
- previousSQLEndLine := int(list[i-1].End.GetLine())
+ // End.Line is 1-based per proto spec, convert to 0-based
+ previousSQLEndLine := int(list[i-1].End.GetLine()) - 1
previousSQLEndColumn := int(list[i-1].End.GetColumn())
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
diff --git a/backend/plugin/parser/tsql/completion_test.go b/backend/plugin/parser/tsql/completion_test.go
index 2287d42fad..356e6bc9c7 100644
--- a/backend/plugin/parser/tsql/completion_test.go
+++ b/backend/plugin/parser/tsql/completion_test.go
@@ -96,7 +96,7 @@ func TestCompletion(t *testing.T) {
if record {
tests[i].Want = filteredResult
} else {
- a.Equalf(t.Want, filteredResult, t.Input, "Case %02d: %s", i, t.Description)
+ a.Equalf(t.Want, filteredResult, "Case %02d: %s", i, t.Input)
}
}