Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(parser): make Start position point to first character of Text (#18694)

h3n4l1725f34f91da
diff --git a/backend/plugin/advisor/tidb/test/column_current_time_count_limit.yaml b/backend/plugin/advisor/tidb/test/column_current_time_count_limit.yaml
index fdfb13f2c5..438d2c561a 100644
--- a/backend/plugin/advisor/tidb/test/column_current_time_count_limit.yaml
+++ b/backend/plugin/advisor/tidb/test/column_current_time_count_limit.yaml
@@ -13,7 +13,7 @@
title: COLUMN_CURRENT_TIME_COUNT_LIMIT
content: Table `t` has 3 DEFAULT CURRENT_TIMESTAMP() columns. The count greater than 2.
startposition:
- line: 5
+ line: 1
column: 0
endposition: null
- statement: |-
@@ -28,7 +28,7 @@
title: COLUMN_CURRENT_TIME_COUNT_LIMIT
content: Table `t` has 2 ON UPDATE CURRENT_TIMESTAMP() columns. The count greater than 1.
startposition:
- line: 4
+ line: 1
column: 0
endposition: null
- statement: |-
diff --git a/backend/plugin/advisor/tidb/test/column_required.yaml b/backend/plugin/advisor/tidb/test/column_required.yaml
index 6a71e65c51..69c2d99333 100644
--- a/backend/plugin/advisor/tidb/test/column_required.yaml
+++ b/backend/plugin/advisor/tidb/test/column_required.yaml
@@ -113,7 +113,7 @@
title: COLUMN_REQUIRED
content: 'Table `book` requires columns: updater_id'
startposition:
- line: 5
+ line: 1
column: 0
endposition: null
- statement: |-
@@ -142,7 +142,7 @@
title: COLUMN_REQUIRED
content: 'Table `book` requires columns: creator_id'
startposition:
- line: 5
+ line: 1
column: 0
endposition: null
- status: 2
@@ -150,7 +150,7 @@
title: COLUMN_REQUIRED
content: 'Table `student` requires columns: creator_id, updater_id'
startposition:
- line: 9
+ line: 6
column: 0
endposition: null
- statement: |-
diff --git a/backend/plugin/advisor/tidb/test/index_total_number_limit.yaml b/backend/plugin/advisor/tidb/test/index_total_number_limit.yaml
index 80dee17c33..1006c21f6c 100644
--- a/backend/plugin/advisor/tidb/test/index_total_number_limit.yaml
+++ b/backend/plugin/advisor/tidb/test/index_total_number_limit.yaml
@@ -52,6 +52,6 @@
title: INDEX_TOTAL_NUMBER_LIMIT
content: The count of index in table `t` should be no more than 5, but found 6
startposition:
- line: 9
+ line: 1
column: 0
endposition: null
diff --git a/backend/plugin/advisor/tidb/test/table_disallow_partition.yaml b/backend/plugin/advisor/tidb/test/table_disallow_partition.yaml
index cca95a356c..bc44903ea6 100644
--- a/backend/plugin/advisor/tidb/test/table_disallow_partition.yaml
+++ b/backend/plugin/advisor/tidb/test/table_disallow_partition.yaml
@@ -16,7 +16,7 @@
PARTITION p1 VALUES LESS THAN (11)
)" creates
startposition:
- line: 4
+ line: 1
column: 0
endposition: null
- statement: |-
@@ -35,6 +35,6 @@
PARTITION p1 VALUES LESS THAN (11)
)" creates
startposition:
- line: 4
+ line: 1
column: 0
endposition: null
diff --git a/backend/plugin/parser/base/lexer.go b/backend/plugin/parser/base/lexer.go
index a1553b568f..fa7271d622 100644
--- a/backend/plugin/parser/base/lexer.go
+++ b/backend/plugin/parser/base/lexer.go
@@ -60,10 +60,13 @@ func IsEmpty(tokens []antlr.Token, semi int) bool {
// SplitSQLByLexer is a grammar-free helper function that splits SQL statements using an ANTLR lexer.
// It works with any ANTLR-based lexer by accepting the token stream and semicolon token type as parameters.
//
+// IMPORTANT: This function requires that the lexer sends whitespace to the hidden channel
+// (using `-> channel(HIDDEN)` in the grammar), NOT skipping it. This ensures GetAllTokens()
+// returns all tokens including whitespace, allowing us to use token positions directly.
+//
// Parameters:
// - stream: The ANTLR token stream (must be already filled with stream.Fill())
// - semiTokenType: The token type value for semicolon in the specific grammar
-// - statement: The original SQL statement string (used for position conversion)
//
// Returns:
// - A slice of Statement, each representing one statement with its text, position, and metadata
@@ -73,11 +76,12 @@ func IsEmpty(tokens []antlr.Token, semi int) bool {
// lexer := parser.NewSnowflakeLexer(antlr.NewInputStream(statement))
// stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
// stream.Fill()
-// return SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI, statement)
-func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, statement string) ([]Statement, error) {
+// return SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI)
+func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int) ([]Statement, error) {
tokens := stream.GetAllTokens()
var buf []antlr.Token
var sqls []Statement
+ byteOffset := 0
for i, token := range tokens {
// Collect all tokens except the last one (EOF)
@@ -101,27 +105,32 @@ func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, stateme
}
}
- // Get the position of the first default channel token for Start position
- antlrPosition := FirstDefaultChannelTokenPosition(buf)
+ stmtText := bufStr.String()
+ stmtByteLength := len(stmtText)
// Use the last token in the buffer for End position (not EOF token when at end of stream)
lastToken := buf[len(buf)-1]
+ // Use buf[0]'s position directly since GetAllTokens() includes hidden channel tokens (whitespace/comments)
sqls = append(sqls, Statement{
- Text: bufStr.String(),
+ Text: stmtText,
BaseLine: buf[0].GetLine() - 1, // BaseLine is the offset of the first token
Range: &storepb.Range{
- Start: int32(buf[0].GetStart()),
- End: int32(lastToken.GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRTokenToExclusiveEndPosition(
int32(lastToken.GetLine()),
int32(lastToken.GetColumn()),
lastToken.GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(buf[0].GetLine()),
+ Column: int32(buf[0].GetColumn() + 1),
+ },
Empty: empty,
})
+ byteOffset += stmtByteLength
buf = nil
continue
}
@@ -129,3 +138,21 @@ func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, stateme
return sqls, nil
}
+
+// CalculateLineAndColumn calculates the 0-based line number and 0-based column (character offset)
+// for a given byte offset in the statement.
+func CalculateLineAndColumn(statement string, byteOffset int) (line, column int) {
+ if byteOffset > len(statement) {
+ byteOffset = len(statement)
+ }
+ // Range over string iterates over runes (code points), not bytes
+ for _, r := range statement[:byteOffset] {
+ if r == '\n' {
+ line++
+ column = 0
+ } else {
+ column++
+ }
+ }
+ return line, column
+}
diff --git a/backend/plugin/parser/bigquery/split.go b/backend/plugin/parser/bigquery/split.go
index 0a86d05160..f296bc7512 100644
--- a/backend/plugin/parser/bigquery/split.go
+++ b/backend/plugin/parser/bigquery/split.go
@@ -1,12 +1,9 @@
package bigquery
import (
- "strings"
-
"github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/parser/googlesql"
- "github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
)
@@ -19,49 +16,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {
lexer := parser.NewGoogleSQLLexer(antlr.NewInputStream(statement))
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
-
- tokens := stream.GetAllTokens()
- var buf []antlr.Token
- var sqls []base.Statement
- byteOffset := 0
- for i, token := range tokens {
- if i < len(tokens)-1 {
- buf = append(buf, token)
- }
- if (token.GetTokenType() == parser.GoogleSQLLexerSEMI_SYMBOL || i == len(tokens)-1) && len(buf) > 0 {
- bufStr := new(strings.Builder)
- empty := true
-
- for _, b := range buf {
- if _, err := bufStr.WriteString(b.GetText()); err != nil {
- return nil, err
- }
- if b.GetChannel() != antlr.TokenHiddenChannel {
- empty = false
- }
- }
- stmtText := bufStr.String()
- stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)
- sqls = append(sqls, base.Statement{
- Text: stmtText,
- BaseLine: buf[0].GetLine() - 1,
- Range: &storepb.Range{
- Start: int32(byteOffset),
- End: int32(byteOffset + stmtByteLength),
- },
- End: common.ConvertANTLRTokenToExclusiveEndPosition(
- int32(buf[len(buf)-1].GetLine()),
- int32(buf[len(buf)-1].GetColumn()),
- buf[len(buf)-1].GetText(),
- ),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
- Empty: empty,
- })
- byteOffset += stmtByteLength
- buf = nil
- continue
- }
- }
- return sqls, nil
+ return base.SplitSQLByLexer(stream, parser.GoogleSQLLexerSEMI_SYMBOL)
}
diff --git a/backend/plugin/parser/bigquery/test-data/test_split.yaml b/backend/plugin/parser/bigquery/test-data/test_split.yaml
index 430a24714b..57c00db271 100644
--- a/backend/plugin/parser/bigquery/test-data/test_split.yaml
+++ b/backend/plugin/parser/bigquery/test-data/test_split.yaml
@@ -1,25 +1,99 @@
-- description: "single statement without semicolon"
- input: "SELECT 1"
+- description: single statement without semicolon
+ input: SELECT 1
result:
- - text: "SELECT 1"
+ - text: SELECT 1
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 9}
- range: {start: 0, end: 8}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 9
+ range:
+ start: 0
+ end: 8
empty: false
-
-- description: "multiple statements multi-line"
+- description: multiple statements multi-line
input: "SELECT 1;\n SELECT\n 33;"
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- text: "\n SELECT\n 33;"
baseline: 0
- start: {line: 2, column: 2}
- end: {line: 3, column: 5}
- range: {start: 9, end: 22}
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 5
+ range:
+ start: 9
+ end: 22
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/cassandra/split.go b/backend/plugin/parser/cassandra/split.go
index 3146b8ceff..411cbc257c 100644
--- a/backend/plugin/parser/cassandra/split.go
+++ b/backend/plugin/parser/cassandra/split.go
@@ -18,5 +18,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
- return base.SplitSQLByLexer(stream, cql.CqlLexerSEMI, statement)
+ return base.SplitSQLByLexer(stream, cql.CqlLexerSEMI)
}
diff --git a/backend/plugin/parser/cassandra/test-data/test_split.yaml b/backend/plugin/parser/cassandra/test-data/test_split.yaml
index 0043d41600..60ded7187e 100644
--- a/backend/plugin/parser/cassandra/test-data/test_split.yaml
+++ b/backend/plugin/parser/cassandra/test-data/test_split.yaml
@@ -1,31 +1,111 @@
-- description: "multiple statements on multiple lines"
+- description: multiple statements on multiple lines
input: "SELECT * FROM users;\n\tSELECT * FROM orders;\n\tSELECT * FROM products;"
result:
- - text: "SELECT * FROM users;"
+ - text: SELECT * FROM users;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 21}
- range: {start: 0, end: 20}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 21
+ range:
+ start: 0
+ end: 20
empty: false
- text: "\n\tSELECT * FROM orders;"
baseline: 0
- start: {line: 2, column: 2}
- end: {line: 2, column: 23}
- range: {start: 20, end: 43}
+ start:
+ line: 1
+ column: 21
+ end:
+ line: 2
+ column: 23
+ range:
+ start: 20
+ end: 43
empty: false
- text: "\n\tSELECT * FROM products;"
baseline: 1
- start: {line: 3, column: 2}
- end: {line: 3, column: 25}
- range: {start: 43, end: 68}
+ start:
+ line: 2
+ column: 23
+ end:
+ line: 3
+ column: 25
+ range:
+ start: 43
+ end: 68
empty: false
-
-- description: "single statement"
- input: "SELECT * FROM users;"
+- description: single statement
+ input: SELECT * FROM users;
result:
- - text: "SELECT * FROM users;"
+ - text: SELECT * FROM users;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 21}
- range: {start: 0, end: 20}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 21
+ range:
+ start: 0
+ end: 20
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/cosmosdb/split.go b/backend/plugin/parser/cosmosdb/split.go
index 2eb1f1f6ae..6afe56f0f8 100644
--- a/backend/plugin/parser/cosmosdb/split.go
+++ b/backend/plugin/parser/cosmosdb/split.go
@@ -46,26 +46,22 @@ func SplitSQL(statement string) ([]base.Statement, error) {
}
}
- var firstToken, lastToken antlr.Token
+ var lastToken antlr.Token
for _, token := range tokens {
if token.GetTokenType() == antlr.TokenEOF {
break
}
- if firstToken == nil && token.GetChannel() == antlr.TokenDefaultChannel {
- firstToken = token
- }
if token.GetChannel() == antlr.TokenDefaultChannel {
lastToken = token
}
}
- if firstToken == nil {
- firstToken = tokens[0]
- }
if lastToken == nil && len(tokens) > 0 {
lastToken = tokens[len(tokens)-1]
}
+ // Since Text is the entire statement (including any leading whitespace),
+ // Start should point to the first character (position 1,1)
return []base.Statement{
{
Text: statement,
@@ -74,10 +70,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {
Start: 0,
End: int32(len(statement)),
},
- Start: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
- Line: int32(firstToken.GetLine()),
- Column: int32(firstToken.GetColumn()),
- }, statement),
+ Start: &storepb.Position{
+ Line: 1,
+ Column: 1,
+ },
End: common.ConvertANTLRTokenToExclusiveEndPosition(
int32(lastToken.GetLine()),
int32(lastToken.GetColumn()),
diff --git a/backend/plugin/parser/cosmosdb/test-data/test_split.yaml b/backend/plugin/parser/cosmosdb/test-data/test_split.yaml
index 8524b067ad..50287e2a78 100644
--- a/backend/plugin/parser/cosmosdb/test-data/test_split.yaml
+++ b/backend/plugin/parser/cosmosdb/test-data/test_split.yaml
@@ -1,29 +1,79 @@
-- description: "single SELECT statement"
- input: "SELECT * FROM c"
+- description: single SELECT statement
+ input: SELECT * FROM c
result:
- - text: "SELECT * FROM c"
+ - text: SELECT * FROM c
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 16}
- range: {start: 0, end: 15}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 16
+ range:
+ start: 0
+ end: 15
empty: false
-
-- description: "SELECT with WHERE"
- input: "SELECT c.id FROM users c WHERE c.active = true"
+- description: SELECT with WHERE
+ input: SELECT c.id FROM users c WHERE c.active = true
result:
- - text: "SELECT c.id FROM users c WHERE c.active = true"
+ - text: SELECT c.id FROM users c WHERE c.active = true
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 47}
- range: {start: 0, end: 46}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 47
+ range:
+ start: 0
+ end: 46
empty: false
-
-- description: "multi-line SELECT"
- input: "SELECT\n * FROM c"
+- description: multi-line SELECT
+ input: |-
+ SELECT
+ * FROM c
result:
- - text: "SELECT\n * FROM c"
+ - text: |-
+ SELECT
+ * FROM c
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 11}
- range: {start: 0, end: 17}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 11
+ range:
+ start: 0
+ end: 17
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1"
+ result:
+ - text: "\n\nSELECT 1"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 9
+ range:
+ start: 0
+ end: 10
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1"
+ result:
+ - text: "\n \n SELECT 1"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 11
+ range:
+ start: 0
+ end: 14
empty: false
diff --git a/backend/plugin/parser/doris/split.go b/backend/plugin/parser/doris/split.go
index 39cd3ef308..7ee49fc059 100644
--- a/backend/plugin/parser/doris/split.go
+++ b/backend/plugin/parser/doris/split.go
@@ -19,5 +19,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
- return base.SplitSQLByLexer(stream, parser.DorisLexerSEMICOLON, statement)
+ return base.SplitSQLByLexer(stream, parser.DorisLexerSEMICOLON)
}
diff --git a/backend/plugin/parser/doris/test-data/test_split.yaml b/backend/plugin/parser/doris/test-data/test_split.yaml
index 3d29e14d8c..7459717ad9 100644
--- a/backend/plugin/parser/doris/test-data/test_split.yaml
+++ b/backend/plugin/parser/doris/test-data/test_split.yaml
@@ -1,51 +1,141 @@
-- description: "simple single statement"
- input: "SELECT 1;"
+- description: simple single statement
+ input: SELECT 1;
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
-
-- description: "multiple statements on one line"
- input: "SELECT 1; SELECT 2;"
+- description: multiple statements on one line
+ input: SELECT 1; SELECT 2;
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- - text: " SELECT 2;"
+ - text: ' SELECT 2;'
baseline: 0
- start: {line: 1, column: 11}
- end: {line: 1, column: 20}
- range: {start: 9, end: 19}
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 1
+ column: 20
+ range:
+ start: 9
+ end: 19
empty: false
-
-- description: "multiple statements on multiple lines"
+- description: multiple statements on multiple lines
input: "SELECT 1;\nSELECT 2;"
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- text: "\nSELECT 2;"
baseline: 0
- start: {line: 2, column: 1}
- end: {line: 2, column: 10}
- range: {start: 9, end: 19}
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 2
+ column: 10
+ range:
+ start: 9
+ end: 19
empty: false
-
-- description: "multi-line statement"
+- description: multi-line statement
input: "SELECT\n 1;"
result:
- text: "SELECT\n 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 5}
- range: {start: 0, end: 11}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 5
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/mysql/mysql.go b/backend/plugin/parser/mysql/mysql.go
index e1f534d6a4..6c182ae53b 100644
--- a/backend/plugin/parser/mysql/mysql.go
+++ b/backend/plugin/parser/mysql/mysql.go
@@ -1,7 +1,6 @@
package mysql
import (
- "fmt"
"log/slog"
"regexp"
"strings"
@@ -95,7 +94,7 @@ func DealWithDelimiter(statement string) (string, error) {
return "", err
}
if has {
- var result []string
+ var result strings.Builder
delimiter := `;`
for _, sql := range list {
if IsDelimiter(sql.Text) {
@@ -103,99 +102,30 @@ func DealWithDelimiter(statement string) (string, error) {
if err != nil {
return "", err
}
- result = append(result, "-- "+sql.Text)
+ // Comment out only the DELIMITER line, preserving all other lines for correct line numbers
+ lines := strings.Split(sql.Text, "\n")
+ for i, line := range lines {
+ if IsDelimiter(line) {
+ lines[i] = "-- " + strings.TrimLeft(line, " \t")
+ break
+ }
+ }
+ result.WriteString(strings.Join(lines, "\n"))
continue
}
- // TODO(rebelice): after deal with delimiter, we may cannot get the right line number, fix it.
if delimiter != ";" && !sql.Empty {
- result = append(result, fmt.Sprintf("%s;", strings.TrimSuffix(sql.Text, delimiter)))
+ result.WriteString(strings.TrimSuffix(sql.Text, delimiter))
+ result.WriteString(";")
} else {
- result = append(result, sql.Text)
+ result.WriteString(sql.Text)
}
}
- statement = strings.Join(result, "\n")
+ statement = result.String()
}
return statement, nil
}
-// RestoreDelimiter tries to restore the delimiter statement which is commented by DealWithDelimiter,
-//
-// Assumes the delimiter comment takes up a single line and matches the syntax `-- DELIMITER ?`,
-// also, one delimiter comment MUST only affect one statement, in the other words, the following statement cause UB:
-// -- DELIMITER ;
-// SELECT 1;
-// SELECT 2;
-// -- DELIMITER ;.
-// If the current delimiter is `;`, and meet `-- DELIMITER ;`, we would skip this delimiter comment.
-func RestoreDelimiter(statement string) (string, error) {
- delimiterRegexp := regexp.MustCompile(`(?i)^-- DELIMITER\s+(?P<DELIMITER>[^\s\\]+)\s*`)
- lines := strings.Split(statement, "\n")
- var result strings.Builder
- var buf strings.Builder
- previousDelimiter := ";"
-
- flushBuf := func(delimiter string) error {
- bufStmt := buf.String()
- buf.Reset()
- // Lookup reversely to modify the last delimiter.
- for i := len(bufStmt) - 1; i >= 0; i-- {
- if bufStmt[i] == ';' {
- bufStmt = bufStmt[:i] + delimiter + bufStmt[i+1:]
- break
- }
- }
- if _, err := result.WriteString(bufStmt); err != nil {
- return errors.Wrapf(err, "failed to write string %q into builder", bufStmt)
- }
- return nil
- }
-
- for i, line := range lines {
- matchList := delimiterRegexp.FindStringSubmatch(line)
- if matchList == nil {
- target := &result
- if previousDelimiter != ";" {
- target = &buf
- }
- // Like strings.Join.
- s := line
- if i != len(lines)-1 {
- s = fmt.Sprintf("%s\n", line)
- }
- if _, err := target.WriteString(s); err != nil {
- return "", errors.Wrapf(err, "failed to write string %q into builder", line)
- }
- continue
- }
- if len(matchList) != 2 {
- return "", errors.Errorf("invalid delimiter comment %q", line)
- }
-
- // If meet delimiter again, we should convert the delimiter in buf to previous delimiter.
- if previousDelimiter != ";" {
- if err := flushBuf(previousDelimiter); err != nil {
- return "", err
- }
- }
-
- if matchList[1] != previousDelimiter {
- if _, err := result.WriteString(fmt.Sprintf("DELIMITER %s\n", matchList[1])); err != nil {
- return "", errors.Wrapf(err, "failed to write string %q into builder", line)
- }
- }
- previousDelimiter = matchList[1]
- }
-
- if previousDelimiter != ";" {
- if err := flushBuf(previousDelimiter); err != nil {
- return "", err
- }
- }
-
- return result.String(), nil
-}
-
func parseSingleStatement(baseLine int, statement string) (antlr.Tree, *antlr.CommonTokenStream, error) {
input := antlr.NewInputStream(statement)
lexer := parser.NewMySQLLexer(input)
diff --git a/backend/plugin/parser/mysql/mysql_test.go b/backend/plugin/parser/mysql/mysql_test.go
index 96f7a129b4..ba7231e0ba 100644
--- a/backend/plugin/parser/mysql/mysql_test.go
+++ b/backend/plugin/parser/mysql/mysql_test.go
@@ -155,300 +155,6 @@ func TestMySQLParser(t *testing.T) {
}
}
-// func TestDealWithDelimiter_SplitCommentBeforeDelimiter(t *testing.T) {
-// tests := []struct {
-// input string
-// want string
-// }{
-// {
-// input: `
-// DELIMITER ;;
-// CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
-// DETERMINISTIC
-// RETURN CONCAT('Hello, ',s,'!') ;;
-// DELIMITER ;
-// --
-// -- Function structure for hello2
-// --
-// DELIMITER ;;
-// CREATE FUNCTION hello2(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
-// DETERMINISTIC
-// RETURN CONCAT('Hello, ',s,'!') ;;
-// DELIMITER ;
-// `,
-// want: `-- DELIMITER ;;
-// CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
-// DETERMINISTIC
-// RETURN CONCAT('Hello, ',s,'!') ;
-// -- DELIMITER ;
-// --
-// -- Function structure for hello2
-// --
-// -- DELIMITER ;;
-// CREATE FUNCTION hello2(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
-// DETERMINISTIC
-// RETURN CONCAT('Hello, ',s,'!') ;
-// -- DELIMITER ;`,
-// },
-// }
-
-// a := require.New(t)
-// for i, test := range tests {
-// got, err := DealWithDelimiter(test.input, tokenizer.SplitCommentBeforeDelimiter())
-// a.NoError(err, i)
-// a.Equal(test.want, got, i)
-// }
-// }
-
-func TestRestoreDelimiter(t *testing.T) {
- testCases := []struct {
- input string
- want string
- }{
- {
- // No delimiter
- input: `
-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
---
--- Table structure for u
---
-CREATE TABLE u (
- b blob NOT NULL,
- t text NOT NULL,
- g geometry NOT NULL,
- j json NOT NULL,
- ti tinyint(1) NOT NULL,
- tb tinyblob
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-`,
- want: `
-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
---
--- Table structure for u
---
-CREATE TABLE u (
- b blob NOT NULL,
- t text NOT NULL,
- g geometry NOT NULL,
- j json NOT NULL,
- ti tinyint(1) NOT NULL,
- tb tinyblob
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-`,
- },
-
- {
- // Semicolon in body after delimiter
- input: `
-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
---
--- Table structure for u
---
-CREATE TABLE u (
- b blob NOT NULL,
- t text NOT NULL,
- g geometry NOT NULL,
- j json NOT NULL,
- ti tinyint(1) NOT NULL,
- tb tinyblob
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-
--- DELIMITER ;;
-CREATE DEFINER=root@localhost PROCEDURE GetEmployeeFullName(IN in_employee_id INT, OUT out_full_name VARCHAR(100))
-BEGIN
- -- Declare variables
- DECLARE first_name VARCHAR(50);
- DECLARE last_name VARCHAR(50);
-
- -- Initialize variables
- SET out_full_name = NULL;
-
- -- Retrieve employee details based on employee_id
- SELECT first_name, last_name INTO first_name, last_name
- FROM employees
- WHERE employee_id = in_employee_id;
-
- -- Check if employee exists
- IF first_name IS NOT NULL THEN
- -- Concatenate first and last names
- SET out_full_name = CONCAT(first_name, ' ', last_name);
- END IF;
-END ;
--- DELIMITER ;
-`,
- want: `
-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
---
--- Table structure for u
---
-CREATE TABLE u (
- b blob NOT NULL,
- t text NOT NULL,
- g geometry NOT NULL,
- j json NOT NULL,
- ti tinyint(1) NOT NULL,
- tb tinyblob
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-
-DELIMITER ;;
-CREATE DEFINER=root@localhost PROCEDURE GetEmployeeFullName(IN in_employee_id INT, OUT out_full_name VARCHAR(100))
-BEGIN
- -- Declare variables
- DECLARE first_name VARCHAR(50);
- DECLARE last_name VARCHAR(50);
-
- -- Initialize variables
- SET out_full_name = NULL;
-
- -- Retrieve employee details based on employee_id
- SELECT first_name, last_name INTO first_name, last_name
- FROM employees
- WHERE employee_id = in_employee_id;
-
- -- Check if employee exists
- IF first_name IS NOT NULL THEN
- -- Concatenate first and last names
- SET out_full_name = CONCAT(first_name, ' ', last_name);
- END IF;
-END ;;
-DELIMITER ;
-`,
- },
- {
- // One delimiter
- input: `
-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
---
--- Table structure for u
---
-CREATE TABLE u (
- b blob NOT NULL,
- t text NOT NULL,
- g geometry NOT NULL,
- j json NOT NULL,
- ti tinyint(1) NOT NULL,
- tb tinyblob
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
--- DELIMITER ;;
-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;
-`,
- want: `
-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
---
--- Table structure for u
---
-CREATE TABLE u (
- b blob NOT NULL,
- t text NOT NULL,
- g geometry NOT NULL,
- j json NOT NULL,
- ti tinyint(1) NOT NULL,
- tb tinyblob
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-DELIMITER ;;
-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;;
-`,
- },
-
- {
- // Multiple delimiters.
- input: `
-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
---
--- Table structure for u
---
-CREATE TABLE u (
- b blob NOT NULL,
- t text NOT NULL,
- g geometry NOT NULL,
- j json NOT NULL,
- ti tinyint(1) NOT NULL,
- tb tinyblob
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
--- DELIMITER ;;
-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;
--- DELIMITER ;
-CREATE FUNCTION hello2(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;
--- DELIMITER ??
-CREATE FUNCTION hello3(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;
--- DELIMITER ;
-DROP FUNCTION hello3;
-`,
- want: `
-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;
-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;
---
--- Table structure for u
---
-CREATE TABLE u (
- b blob NOT NULL,
- t text NOT NULL,
- g geometry NOT NULL,
- j json NOT NULL,
- ti tinyint(1) NOT NULL,
- tb tinyblob
-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;
-DELIMITER ;;
-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;;
-DELIMITER ;
-CREATE FUNCTION hello2(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;
-DELIMITER ??
-CREATE FUNCTION hello3(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ??
-DELIMITER ;
-DROP FUNCTION hello3;
-`,
- },
-
- // Ignore redundant DELIMITER statements.
- {
- input: `
--- DELIMITER ;;
-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;
--- DELIMITER ;
--- DELIMITER ;
-`,
- want: `
-DELIMITER ;;
-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4
- DETERMINISTIC
-RETURN CONCAT('Hello, ',s,'!') ;;
-DELIMITER ;
-`,
- },
- }
-
- for i, tc := range testCases {
- got, err := RestoreDelimiter(tc.input)
- require.NoError(t, err)
- require.Equalf(t, tc.want, got, "test cases: %d", i)
- }
-}
-
func TestParseMySQLStatements(t *testing.T) {
statement := "SELECT 1; SELECT 2;"
diff --git a/backend/plugin/parser/mysql/split.go b/backend/plugin/parser/mysql/split.go
index 84a5e1885d..7d943a51f7 100644
--- a/backend/plugin/parser/mysql/split.go
+++ b/backend/plugin/parser/mysql/split.go
@@ -66,7 +66,8 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
stmtStartByte := len(statement[:stmtStartChar])
stmtEndByte := len(statement[:stmtEndChar])
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : i+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, stmtStartByte)
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
@@ -81,7 +82,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
int32(tokens[i].GetColumn()),
tokens[i].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:i+1], parser.MySQLLexerSEMICOLON_SYMBOL),
})
i++
@@ -103,7 +107,8 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
stmtStartByte := len(statement[:stmtStartChar])
stmtEndByte := len(statement[:stmtEndChar])
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:newStart])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, stmtStartByte)
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
@@ -119,7 +124,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
int32(tokens[newStart-1].GetColumn()),
tokens[newStart-1].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:i], parser.MySQLLexerSEMICOLON_SYMBOL),
})
i = newStart
@@ -140,7 +148,8 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
stmtStartByte := len(statement[:stmtStartChar])
stmtEndByte := len(statement[:stmtEndChar])
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, stmtStartByte)
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
@@ -155,7 +164,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
int32(tokens[endPos-1].GetColumn()),
tokens[endPos-1].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:endPos], parser.MySQLLexerSEMICOLON_SYMBOL),
})
}
@@ -241,7 +253,8 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Com
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
@@ -256,7 +269,10 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Com
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),
})
byteOffset += stmtByteLength
@@ -268,7 +284,8 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Com
stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
@@ -283,7 +300,10 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Com
int32(tokens[eofPos-1].GetColumn()),
tokens[eofPos-1].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:eofPos], parser.MySQLLexerSEMICOLON_SYMBOL),
})
}
@@ -437,7 +457,8 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
@@ -452,7 +473,10 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),
})
byteOffset += stmtByteLength
@@ -464,7 +488,8 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b
stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
@@ -479,7 +504,10 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b
int32(tokens[eofPos-1].GetColumn()),
tokens[eofPos-1].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:eofPos], parser.MySQLLexerSEMICOLON_SYMBOL),
})
}
diff --git a/backend/plugin/parser/mysql/test-data/test_backup.yaml b/backend/plugin/parser/mysql/test-data/test_backup.yaml
index 08ba863637..d6f6ae0d69 100644
--- a/backend/plugin/parser/mysql/test-data/test_backup.yaml
+++ b/backend/plugin/parser/mysql/test-data/test_backup.yaml
@@ -160,8 +160,8 @@
sourcetablename: t2
targettablename: _rollback_t2_db2
startposition:
- line: 2
- column: 1
+ line: 1
+ column: 39
endposition:
line: 2
column: 39
diff --git a/backend/plugin/parser/mysql/test-data/test_split.yaml b/backend/plugin/parser/mysql/test-data/test_split.yaml
index 2c2b35db29..84c8204ea0 100644
--- a/backend/plugin/parser/mysql/test-data/test_split.yaml
+++ b/backend/plugin/parser/mysql/test-data/test_split.yaml
@@ -1,243 +1,480 @@
-- description: "DELIMITER with stored procedure"
- input: "\n\t\t\tDELIMITER ;;\n\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\tBEGIN\n\t\t\t\tDECLARE x INT;\n\t\t\t\tSET x = 0;\n\t\t\t\tlabel1: WHILE x < p1 DO\n\t\t\t\t\tSET x = x + 1;\n\t\t\t\tEND WHILE label1;\n\t\t\tEND;;\n\t\t\tDELIMITER ;\n\t\t\tCALL dorepeat(1000);\n\t\t\tSELECT x;\n\t\t\t"
+- description: DELIMITER with stored procedure
+ input: "\t\t\tDELIMITER ;;\n\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\tBEGIN\n\t\t\t\tDECLARE x INT;\n\t\t\t\tSET x = 0;\n\t\t\t\tlabel1: WHILE x < p1 DO\n\t\t\t\t\tSET x = x + 1;\n\t\t\t\tEND WHILE label1;\n\t\t\tEND;;\n\t\t\tDELIMITER ;\n\t\t\tCALL dorepeat(1000);\n\t\t\tSELECT x;\n\t\t\t"
result:
- text: "\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\tBEGIN\n\t\t\t\tDECLARE x INT;\n\t\t\t\tSET x = 0;\n\t\t\t\tlabel1: WHILE x < p1 DO\n\t\t\t\t\tSET x = x + 1;\n\t\t\t\tEND WHILE label1;\n\t\t\tEND;"
- baseline: 2
- start: {line: 3, column: 4}
- end: {line: 10, column: 9}
- range: {start: 17, end: 175}
+ baseline: 1
+ start:
+ line: 2
+ column: 1
+ end:
+ line: 9
+ column: 9
+ range:
+ start: 16
+ end: 174
empty: false
- text: "\t\t\tCALL dorepeat(1000);"
- baseline: 11
- start: {line: 12, column: 4}
- end: {line: 12, column: 24}
- range: {start: 191, end: 214}
+ baseline: 10
+ start:
+ line: 11
+ column: 1
+ end:
+ line: 11
+ column: 24
+ range:
+ start: 190
+ end: 213
empty: false
- text: "\n\t\t\tSELECT x;"
- baseline: 11
- start: {line: 13, column: 4}
- end: {line: 13, column: 13}
- range: {start: 214, end: 227}
+ baseline: 10
+ start:
+ line: 11
+ column: 24
+ end:
+ line: 12
+ column: 13
+ range:
+ start: 213
+ end: 226
empty: false
- text: "\n\t\t\t"
- baseline: 12
- start: {line: 14, column: 4}
- end: {line: 14, column: 4}
- range: {start: 227, end: 231}
+ baseline: 11
+ start:
+ line: 12
+ column: 13
+ end:
+ line: 13
+ column: 4
+ range:
+ start: 226
+ end: 230
empty: true
-
-- description: "escaped double quote in string"
- input: "select * from t;select \"\\\"\" where true;"
+- description: escaped double quote in string
+ input: select * from t;select "\"" where true;
result:
- - text: "select * from t;"
+ - text: select * from t;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 17}
- range: {start: 0, end: 16}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 17
+ range:
+ start: 0
+ end: 16
empty: false
- - text: "select \"\\\"\" where true;"
+ - text: select "\"" where true;
baseline: 0
- start: {line: 1, column: 17}
- end: {line: 1, column: 40}
- range: {start: 16, end: 39}
+ start:
+ line: 1
+ column: 17
+ end:
+ line: 1
+ column: 40
+ range:
+ start: 16
+ end: 39
empty: false
-
-- description: "comments only creates empty statement"
+- description: comments only creates empty statement
input: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n"
result:
- text: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n"
baseline: 0
- start: {line: 3, column: 1}
- end: {line: 2, column: 24}
- range: {start: 0, end: 39}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 24
+ range:
+ start: 0
+ end: 39
empty: true
-
-- description: "statement followed by trailing block comment"
+- description: statement followed by trailing block comment
input: "select * from t;\n\t\t\t/* sdfasdf */"
result:
- - text: "select * from t;"
+ - text: select * from t;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 17}
- range: {start: 0, end: 16}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 17
+ range:
+ start: 0
+ end: 16
empty: false
- text: "\n\t\t\t/* sdfasdf */"
baseline: 0
- start: {line: 2, column: 17}
- end: {line: 2, column: 17}
- range: {start: 16, end: 33}
+ start:
+ line: 1
+ column: 17
+ end:
+ line: 2
+ column: 17
+ range:
+ start: 16
+ end: 33
empty: true
-
-- description: "statement then empty comment statement then statement"
+- description: statement then empty comment statement then statement
input: "select * from t;\n\t\t\t/* sdfasdf */;\n\t\t\tselect * from t;"
result:
- - text: "select * from t;"
+ - text: select * from t;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 17}
- range: {start: 0, end: 16}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 17
+ range:
+ start: 0
+ end: 16
empty: false
- text: "\n\t\t\t/* sdfasdf */;"
baseline: 0
- start: {line: 2, column: 17}
- end: {line: 2, column: 18}
- range: {start: 16, end: 34}
+ start:
+ line: 1
+ column: 17
+ end:
+ line: 2
+ column: 18
+ range:
+ start: 16
+ end: 34
empty: true
- text: "\n\t\t\tselect * from t;"
baseline: 1
- start: {line: 3, column: 4}
- end: {line: 3, column: 20}
- range: {start: 34, end: 54}
+ start:
+ line: 2
+ column: 18
+ end:
+ line: 3
+ column: 20
+ range:
+ start: 34
+ end: 54
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
-
-- description: "function with BEGIN END block"
+- description: function with BEGIN END block
input: "CREATE DEFINER=`root`@`%` FUNCTION `CalcIncome`( starting_value INT ) RETURNS int\nBEGIN\n\n\t\t DECLARE income INT;\n\n\t\t SET income = 0;\n\n\t\t label1: WHILE income <= 3000 DO\n\t\t\t SET income = income + starting_value;\n\t\t END WHILE label1;\n\n\t\t RETURN income;\n\n\t\tEND ;"
result:
- text: "CREATE DEFINER=`root`@`%` FUNCTION `CalcIncome`( starting_value INT ) RETURNS int\nBEGIN\n\n\t\t DECLARE income INT;\n\n\t\t SET income = 0;\n\n\t\t label1: WHILE income <= 3000 DO\n\t\t\t SET income = income + starting_value;\n\t\t END WHILE label1;\n\n\t\t RETURN income;\n\n\t\tEND ;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 14, column: 8}
- range: {start: 0, end: 268}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 14
+ column: 8
+ range:
+ start: 0
+ end: 268
empty: false
-
-- description: "multiple statements with leading whitespace and no trailing semicolon"
- input: " CREATE TABLE t(a int); CREATE TABLE t1(a int)"
+- description: multiple statements with leading whitespace and no trailing semicolon
+ input: ' CREATE TABLE t(a int); CREATE TABLE t1(a int)'
result:
- - text: " CREATE TABLE t(a int);"
+ - text: ' CREATE TABLE t(a int);'
baseline: 0
- start: {line: 1, column: 5}
- end: {line: 1, column: 27}
- range: {start: 0, end: 26}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 27
+ range:
+ start: 0
+ end: 26
empty: false
- - text: " CREATE TABLE t1(a int)"
+ - text: ' CREATE TABLE t1(a int)'
baseline: 0
- start: {line: 1, column: 28}
- end: {line: 1, column: 50}
- range: {start: 26, end: 49}
+ start:
+ line: 1
+ column: 27
+ end:
+ line: 1
+ column: 50
+ range:
+ start: 26
+ end: 49
empty: false
-
-- description: "backtick identifiers with escaped strings"
+- description: backtick identifiers with escaped strings
input: "CREATE TABLE `tech_Book`(id int, name varchar(255));\nINSERT INTO `tech_Book` VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"
result:
- - text: "CREATE TABLE `tech_Book`(id int, name varchar(255));"
+ - text: CREATE TABLE `tech_Book`(id int, name varchar(255));
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 53}
- range: {start: 0, end: 52}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 53
+ range:
+ start: 0
+ end: 52
empty: false
- text: "\nINSERT INTO `tech_Book` VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"
baseline: 0
- start: {line: 2, column: 1}
- end: {line: 2, column: 79}
- range: {start: 52, end: 131}
+ start:
+ line: 1
+ column: 53
+ end:
+ line: 2
+ column: 79
+ range:
+ start: 52
+ end: 131
empty: false
-
-- description: "block comment, line comment, and hash comment"
- input: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');\n\t\t\t\t\t\t# this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"
+- description: block comment, line comment, and hash comment
+ input: "\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');\n\t\t\t\t\t\t# this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"
result:
- - text: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));"
+ - text: "\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));"
baseline: 0
- start: {line: 3, column: 7}
- end: {line: 3, column: 77}
- range: {start: 0, end: 110}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 77
+ range:
+ start: 0
+ end: 109
empty: false
- text: "\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"
- baseline: 2
- start: {line: 5, column: 7}
- end: {line: 5, column: 83}
- range: {start: 110, end: 223}
+ baseline: 1
+ start:
+ line: 2
+ column: 77
+ end:
+ line: 4
+ column: 83
+ range:
+ start: 109
+ end: 222
empty: false
- text: "\n\t\t\t\t\t\t# this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"
- baseline: 4
- start: {line: 7, column: 7}
- end: {line: 7, column: 83}
- range: {start: 223, end: 335}
+ baseline: 3
+ start:
+ line: 4
+ column: 83
+ end:
+ line: 6
+ column: 83
+ range:
+ start: 222
+ end: 334
empty: false
-
-- description: "procedure with REPEAT and hash comment"
+- description: procedure with REPEAT and hash comment
input: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;\n\t\t\t\t\t\tCALL dorepeat(1000);\n\t\t\t\t\t\tSELECT @x;\n\t\t\t\t\t\t"
result:
- text: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;"
baseline: 0
- start: {line: 2, column: 7}
- end: {line: 7, column: 8}
- range: {start: 0, end: 181}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 7
+ column: 8
+ range:
+ start: 0
+ end: 181
empty: false
- text: "\n\t\t\t\t\t\tCALL dorepeat(1000);"
baseline: 6
- start: {line: 8, column: 7}
- end: {line: 8, column: 27}
- range: {start: 181, end: 208}
+ start:
+ line: 7
+ column: 8
+ end:
+ line: 8
+ column: 27
+ range:
+ start: 181
+ end: 208
empty: false
- text: "\n\t\t\t\t\t\tSELECT @x;"
baseline: 7
- start: {line: 9, column: 7}
- end: {line: 9, column: 17}
- range: {start: 208, end: 225}
+ start:
+ line: 8
+ column: 27
+ end:
+ line: 9
+ column: 17
+ range:
+ start: 208
+ end: 225
empty: false
- text: "\n\t\t\t\t\t\t"
baseline: 8
- start: {line: 10, column: 7}
- end: {line: 10, column: 7}
- range: {start: 225, end: 232}
+ start:
+ line: 9
+ column: 17
+ end:
+ line: 10
+ column: 7
+ range:
+ start: 225
+ end: 232
empty: true
-
-- description: "procedure with block comment inside"
+- description: procedure with block comment inside
input: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\t/* This is a comment */\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;\n\t\t\t\t\t\tCALL dorepeat(1000);\n\t\t\t\t\t\tSELECT @x;\n\t\t\t\t\t\t"
result:
- text: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\t/* This is a comment */\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;"
baseline: 0
- start: {line: 2, column: 7}
- end: {line: 8, column: 8}
- range: {start: 0, end: 211}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 8
+ column: 8
+ range:
+ start: 0
+ end: 211
empty: false
- text: "\n\t\t\t\t\t\tCALL dorepeat(1000);"
baseline: 7
- start: {line: 9, column: 7}
- end: {line: 9, column: 27}
- range: {start: 211, end: 238}
+ start:
+ line: 8
+ column: 8
+ end:
+ line: 9
+ column: 27
+ range:
+ start: 211
+ end: 238
empty: false
- text: "\n\t\t\t\t\t\tSELECT @x;"
baseline: 8
- start: {line: 10, column: 7}
- end: {line: 10, column: 17}
- range: {start: 238, end: 255}
+ start:
+ line: 9
+ column: 27
+ end:
+ line: 10
+ column: 17
+ range:
+ start: 238
+ end: 255
empty: false
- text: "\n\t\t\t\t\t\t"
baseline: 9
- start: {line: 11, column: 7}
- end: {line: 11, column: 7}
- range: {start: 255, end: 262}
+ start:
+ line: 10
+ column: 17
+ end:
+ line: 11
+ column: 7
+ range:
+ start: 255
+ end: 262
empty: true
-
-- description: "CRLF line endings"
+- description: CRLF line endings
input: "CREATE TABLE t\r\n(a int);\r\nCREATE TABLE t1(b int);"
result:
- text: "CREATE TABLE t\r\n(a int);"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 9}
- range: {start: 0, end: 24}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 9
+ range:
+ start: 0
+ end: 24
empty: false
- text: "\r\nCREATE TABLE t1(b int);"
baseline: 1
- start: {line: 3, column: 1}
- end: {line: 3, column: 24}
- range: {start: 24, end: 49}
+ start:
+ line: 2
+ column: 9
+ end:
+ line: 3
+ column: 24
+ range:
+ start: 24
+ end: 49
empty: false
-
-- description: "unicode in table names"
- input: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);"
+- description: unicode in table names
+ input: "SELECT * FROM \u8868\u540D; INSERT INTO \u8868 VALUES (1);"
result:
- - text: "SELECT * FROM 表名;"
+ - text: "SELECT * FROM \u8868\u540D;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 18}
- range: {start: 0, end: 21}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 18
+ range:
+ start: 0
+ end: 21
empty: false
- - text: " INSERT INTO 表 VALUES (1);"
+ - text: " INSERT INTO \u8868 VALUES (1);"
baseline: 0
- start: {line: 1, column: 19}
- end: {line: 1, column: 44}
- range: {start: 21, end: 49}
+ start:
+ line: 1
+ column: 18
+ end:
+ line: 1
+ column: 44
+ range:
+ start: 21
+ end: 49
empty: false
diff --git a/backend/plugin/parser/partiql/completion.go b/backend/plugin/parser/partiql/completion.go
index 55ce5f8073..d0b033def7 100644
--- a/backend/plugin/parser/partiql/completion.go
+++ b/backend/plugin/parser/partiql/completion.go
@@ -569,8 +569,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
// The caret is in the same line as the last line of the previous SQL statement.
- // We need to adjust the caret offset.
- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.
+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.
+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)
+ newCaretOffset = caretOffset - previousSQLEndColumn + 1
}
break
}
diff --git a/backend/plugin/parser/partiql/split.go b/backend/plugin/parser/partiql/split.go
index b7f59a73d1..c5fc67ecd6 100644
--- a/backend/plugin/parser/partiql/split.go
+++ b/backend/plugin/parser/partiql/split.go
@@ -18,5 +18,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
- return base.SplitSQLByLexer(stream, parser.PartiQLLexerCOLON_SEMI, statement)
+ return base.SplitSQLByLexer(stream, parser.PartiQLLexerCOLON_SEMI)
}
diff --git a/backend/plugin/parser/partiql/test-data/test_split.yaml b/backend/plugin/parser/partiql/test-data/test_split.yaml
index 3962b333cb..477ea57888 100644
--- a/backend/plugin/parser/partiql/test-data/test_split.yaml
+++ b/backend/plugin/parser/partiql/test-data/test_split.yaml
@@ -1,33 +1,120 @@
-- description: "multiple DML statements"
+- description: multiple DML statements
input: "DELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money';\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money'};\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money02'};\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money03'};\n\tDELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money02'"
result:
- text: "DELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money';"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 71}
- range: {start: 0, end: 70}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 71
+ range:
+ start: 0
+ end: 70
empty: false
- text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money'};"
baseline: 0
- start: {line: 2, column: 2}
- end: {line: 2, column: 131}
- range: {start: 70, end: 201}
+ start:
+ line: 1
+ column: 71
+ end:
+ line: 2
+ column: 131
+ range:
+ start: 70
+ end: 201
empty: false
- text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money02'};"
baseline: 1
- start: {line: 3, column: 2}
- end: {line: 3, column: 133}
- range: {start: 201, end: 334}
+ start:
+ line: 2
+ column: 131
+ end:
+ line: 3
+ column: 133
+ range:
+ start: 201
+ end: 334
empty: false
- text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money03'};"
baseline: 2
- start: {line: 4, column: 2}
- end: {line: 4, column: 133}
- range: {start: 334, end: 467}
+ start:
+ line: 3
+ column: 133
+ end:
+ line: 4
+ column: 133
+ range:
+ start: 334
+ end: 467
empty: false
- text: "\n\tDELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money02'"
baseline: 3
- start: {line: 5, column: 2}
- end: {line: 5, column: 73}
- range: {start: 467, end: 540}
+ start:
+ line: 4
+ column: 133
+ end:
+ line: 5
+ column: 73
+ range:
+ start: 467
+ end: 540
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.go
index 512c350b6f..44cffcb7f6 100644
--- a/backend/plugin/parser/pg/split.go
+++ b/backend/plugin/parser/pg/split.go
@@ -70,9 +70,8 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antl
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -85,7 +84,10 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antl
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),
})
byteOffset += stmtByteLength
@@ -97,9 +99,8 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antl
stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -112,7 +113,10 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antl
int32(tokens[eofPos-1].GetColumn()),
tokens[eofPos-1].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:eofPos], parser.PostgreSQLParserSEMI),
})
}
@@ -207,9 +211,8 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -222,7 +225,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),
})
byteOffset += stmtByteLength
@@ -234,9 +240,8 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -249,7 +254,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
int32(tokens[eofPos-1].GetColumn()),
tokens[eofPos-1].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:eofPos], parser.PostgreSQLParserSEMI),
})
}
diff --git a/backend/plugin/parser/pg/test-data/test_split.yaml b/backend/plugin/parser/pg/test-data/test_split.yaml
index 92e96fec00..e3f0a9b6c7 100644
--- a/backend/plugin/parser/pg/test-data/test_split.yaml
+++ b/backend/plugin/parser/pg/test-data/test_split.yaml
@@ -1,357 +1,624 @@
-- description: "single statement with semicolon"
+- description: single statement with semicolon
input: "SELECT * FROM users;"
result:
- text: "SELECT * FROM users;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 21}
- range: {start: 0, end: 20}
- empty: false
-
-- description: "single statement without semicolon"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 21
+ range:
+ start: 0
+ end: 20
+ empty: false
+- description: single statement without semicolon
input: "SELECT * FROM users"
result:
- text: "SELECT * FROM users"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 20}
- range: {start: 0, end: 19}
- empty: false
-
-- description: "multiple statements same line"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 20
+ range:
+ start: 0
+ end: 19
+ empty: false
+- description: multiple statements same line
input: "SELECT 1; SELECT 2;"
result:
- text: "SELECT 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- text: " SELECT 2;"
baseline: 0
- start: {line: 1, column: 11}
- end: {line: 1, column: 20}
- range: {start: 9, end: 19}
- empty: false
-
-- description: "leading whitespace preserved"
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 1
+ column: 20
+ range:
+ start: 9
+ end: 19
+ empty: false
+- description: leading whitespace preserved
input: " SELECT 1; SELECT 2;"
result:
- text: " SELECT 1;"
baseline: 0
- start: {line: 1, column: 5}
- end: {line: 1, column: 14}
- range: {start: 0, end: 13}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 14
+ range:
+ start: 0
+ end: 13
empty: false
- text: " SELECT 2;"
baseline: 0
- start: {line: 1, column: 16}
- end: {line: 1, column: 25}
- range: {start: 13, end: 24}
- empty: false
-
-- description: "unicode in table name and values"
+ start:
+ line: 1
+ column: 14
+ end:
+ line: 1
+ column: 25
+ range:
+ start: 13
+ end: 24
+ empty: false
+- description: unicode in table name and values
input: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);"
result:
- text: "SELECT * FROM 表名;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 18}
- range: {start: 0, end: 21}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 18
+ range:
+ start: 0
+ end: 21
empty: false
- text: " INSERT INTO 表 VALUES (1);"
baseline: 0
- start: {line: 1, column: 19}
- end: {line: 1, column: 44}
- range: {start: 21, end: 49}
- empty: false
-
-- description: "simple dollar-quoted string"
+ start:
+ line: 1
+ column: 18
+ end:
+ line: 1
+ column: 44
+ range:
+ start: 21
+ end: 49
+ empty: false
+- description: simple dollar-quoted string
input: "SELECT $$hello; world$$;"
result:
- text: "SELECT $$hello; world$$;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 25}
- range: {start: 0, end: 24}
- empty: false
-
-- description: "tagged dollar-quoted string"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 25
+ range:
+ start: 0
+ end: 24
+ empty: false
+- description: tagged dollar-quoted string
input: "SELECT $tag$hello; world$tag$;"
result:
- text: "SELECT $tag$hello; world$tag$;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 31}
- range: {start: 0, end: 30}
- empty: false
-
-- description: "dollar-quote does not split on internal semicolon"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 31
+ range:
+ start: 0
+ end: 30
+ empty: false
+- description: dollar-quote does not split on internal semicolon
input: "SELECT $$a;b$$; SELECT 2;"
result:
- text: "SELECT $$a;b$$;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 16}
- range: {start: 0, end: 15}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 16
+ range:
+ start: 0
+ end: 15
empty: false
- text: " SELECT 2;"
baseline: 0
- start: {line: 1, column: 17}
- end: {line: 1, column: 26}
- range: {start: 15, end: 25}
- empty: false
-
-- description: "CRLF line endings"
+ start:
+ line: 1
+ column: 16
+ end:
+ line: 1
+ column: 26
+ range:
+ start: 15
+ end: 25
+ empty: false
+- description: CRLF line endings
input: "SELECT 1;\r\nSELECT 2;"
result:
- text: "SELECT 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- text: "\r\nSELECT 2;"
baseline: 0
- start: {line: 2, column: 1}
- end: {line: 2, column: 10}
- range: {start: 9, end: 20}
- empty: false
-
-- description: "multi-line with CRLF"
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 2
+ column: 10
+ range:
+ start: 9
+ end: 20
+ empty: false
+- description: multi-line with CRLF
input: "CREATE TABLE t\r\n(a int);\r\nCREATE TABLE t1(b int);"
result:
- text: "CREATE TABLE t\r\n(a int);"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 9}
- range: {start: 0, end: 24}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 9
+ range:
+ start: 0
+ end: 24
empty: false
- text: "\r\nCREATE TABLE t1(b int);"
baseline: 1
- start: {line: 3, column: 1}
- end: {line: 3, column: 24}
- range: {start: 24, end: 49}
- empty: false
-
-- description: "procedure with dollar-quoted body"
- input: |-
- CREATE PROCEDURE insert_data(a varchar(50))
- LANGUAGE SQL
- AS $$
- INSERT INTO tbl VALUES ('test;value');
- $$;
- SELECT 1;
+ start:
+ line: 2
+ column: 9
+ end:
+ line: 3
+ column: 24
+ range:
+ start: 24
+ end: 49
+ empty: false
+- description: procedure with dollar-quoted body
+ input: "CREATE PROCEDURE insert_data(a varchar(50))\nLANGUAGE SQL\nAS $$\n INSERT INTO tbl VALUES ('test;value');\n$$;\nSELECT 1;"
result:
- text: "CREATE PROCEDURE insert_data(a varchar(50))\nLANGUAGE SQL\nAS $$\n INSERT INTO tbl VALUES ('test;value');\n$$;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 5, column: 4}
- range: {start: 0, end: 107}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 5
+ column: 4
+ range:
+ start: 0
+ end: 107
empty: false
- text: "\nSELECT 1;"
baseline: 4
- start: {line: 6, column: 1}
- end: {line: 6, column: 10}
- range: {start: 107, end: 117}
- empty: false
-
-- description: "function with tagged dollar-quote"
- input: |-
- CREATE FUNCTION get_data()
- RETURNS TEXT
- AS $func$
- SELECT 'value with $$ inside';
- $func$;
- SELECT 2;
+ start:
+ line: 5
+ column: 4
+ end:
+ line: 6
+ column: 10
+ range:
+ start: 107
+ end: 117
+ empty: false
+- description: function with tagged dollar-quote
+ input: "CREATE FUNCTION get_data()\nRETURNS TEXT\nAS $func$\n SELECT 'value with $$ inside';\n$func$;\nSELECT 2;"
result:
- text: "CREATE FUNCTION get_data()\nRETURNS TEXT\nAS $func$\n SELECT 'value with $$ inside';\n$func$;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 5, column: 8}
- range: {start: 0, end: 90}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 5
+ column: 8
+ range:
+ start: 0
+ end: 90
empty: false
- text: "\nSELECT 2;"
baseline: 4
- start: {line: 6, column: 1}
- end: {line: 6, column: 10}
- range: {start: 90, end: 100}
- empty: false
-
-- description: "comments only creates empty statement"
+ start:
+ line: 5
+ column: 8
+ end:
+ line: 6
+ column: 10
+ range:
+ start: 90
+ end: 100
+ empty: false
+- description: comments only creates empty statement
input: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n\t\t\t"
result:
- text: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n\t\t\t"
baseline: 0
- start: {line: 3, column: 4}
- end: {line: 3, column: 4}
- range: {start: 0, end: 42}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 4
+ range:
+ start: 0
+ end: 42
empty: true
-
-- description: "statement followed by trailing comment"
+- description: statement followed by trailing comment
input: "select * from t;\n\t\t\t/* sdfasdf */"
result:
- text: "select * from t;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 17}
- range: {start: 0, end: 16}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 17
+ range:
+ start: 0
+ end: 16
empty: false
- text: "\n\t\t\t/* sdfasdf */"
baseline: 0
- start: {line: 2, column: 17}
- end: {line: 2, column: 17}
- range: {start: 16, end: 33}
+ start:
+ line: 1
+ column: 17
+ end:
+ line: 2
+ column: 17
+ range:
+ start: 16
+ end: 33
empty: true
-
-- description: "statement then empty comment statement then statement"
+- description: statement then empty comment statement then statement
input: "select * from t;\n\t\t\t/* sdfasdf */;\n\t\t\tselect * from t;"
result:
- text: "select * from t;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 17}
- range: {start: 0, end: 16}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 17
+ range:
+ start: 0
+ end: 16
empty: false
- text: "\n\t\t\t/* sdfasdf */;"
baseline: 0
- start: {line: 2, column: 17}
- end: {line: 2, column: 18}
- range: {start: 16, end: 34}
+ start:
+ line: 1
+ column: 17
+ end:
+ line: 2
+ column: 18
+ range:
+ start: 16
+ end: 34
empty: true
- text: "\n\t\t\tselect * from t;"
baseline: 1
- start: {line: 3, column: 4}
- end: {line: 3, column: 20}
- range: {start: 34, end: 54}
- empty: false
-
-- description: "double-quoted identifiers with special characters"
+ start:
+ line: 2
+ column: 18
+ end:
+ line: 3
+ column: 20
+ range:
+ start: 34
+ end: 54
+ empty: false
+- description: double-quoted identifiers with special characters
input: "CREATE TABLE \"tech_Book\"(id int, name varchar(255));\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"
result:
- text: "CREATE TABLE \"tech_Book\"(id int, name varchar(255));"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 53}
- range: {start: 0, end: 52}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 53
+ range:
+ start: 0
+ end: 52
empty: false
- text: "\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"
baseline: 0
- start: {line: 2, column: 7}
- end: {line: 2, column: 83}
- range: {start: 52, end: 135}
- empty: false
-
-- description: "block comment and inline comment with statement"
- input: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"
+ start:
+ line: 1
+ column: 53
+ end:
+ line: 2
+ column: 83
+ range:
+ start: 52
+ end: 135
+ empty: false
+- description: block comment and inline comment with statement
+ input: "\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"
result:
- - text: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));"
- baseline: 0
- start: {line: 3, column: 7}
- end: {line: 3, column: 79}
- range: {start: 0, end: 112}
+ - text: "\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 79
+ range:
+ start: 0
+ end: 111
empty: false
- text: "\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"
- baseline: 2
- start: {line: 5, column: 7}
- end: {line: 5, column: 83}
- range: {start: 112, end: 225}
- empty: false
-
-- description: "unterminated single quote"
+ baseline: 1
+ start:
+ line: 2
+ column: 79
+ end:
+ line: 4
+ column: 83
+ range:
+ start: 111
+ end: 224
+ empty: false
+- description: unterminated single quote
input: "INSERT INTO t VALUES ('klajfas)"
result:
- text: "INSERT INTO t VALUES ('klajfas)"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 32}
- range: {start: 0, end: 31}
- empty: false
-
-- description: "unterminated double quote"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 32
+ range:
+ start: 0
+ end: 31
+ empty: false
+- description: unterminated double quote
input: "INSERT INTO \"t VALUES ('klajfas)"
result:
- text: "INSERT INTO \"t VALUES ('klajfas)"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 33}
- range: {start: 0, end: 32}
- empty: false
-
-- description: "unterminated block comment"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 33
+ range:
+ start: 0
+ end: 32
+ empty: false
+- description: unterminated block comment
input: "/*INSERT INTO \"t VALUES ('klajfas)"
result:
- text: "/*INSERT INTO \"t VALUES ('klajfas)"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 35}
- range: {start: 0, end: 34}
- empty: false
-
-- description: "unterminated dollar quote"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 35
+ range:
+ start: 0
+ end: 34
+ empty: false
+- description: unterminated dollar quote
input: "$$INSERT INTO \"t VALUES ('klajfas)"
result:
- text: "$$INSERT INTO \"t VALUES ('klajfas)"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 35}
- range: {start: 0, end: 34}
- empty: false
-
-- description: "complex procedure with comments and escapes inside dollar-quote"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 35
+ range:
+ start: 0
+ end: 34
+ empty: false
+- description: complex procedure with comments and escapes inside dollar-quote
input: "CREATE PROCEDURE insert_data(a varchar(50), b varchar(50))\n\t\t\t\t\t\tLANGUAGE SQL\n\t\t\t\t\t\tAS $$\n\t\t\t\t\t\t/*this is the comment */\n\t\t\t\t\t\tINSERT /* inline comment */ INTO tbl VALUES ('lkjafd''lksjadlf;lk\\\\jasdflkasdf\"asdklf\\\\');\n\t\t\t\t\t\t-- this is the comment\n\t\t\t\t\t\tINSERT INTO tbl VALUES ('fasf_bkdjlfa');\n\t\t\t\t\t\t$$;\n\t\t\t\t\t\tCREATE TABLE t(a int);"
result:
- text: "CREATE PROCEDURE insert_data(a varchar(50), b varchar(50))\n\t\t\t\t\t\tLANGUAGE SQL\n\t\t\t\t\t\tAS $$\n\t\t\t\t\t\t/*this is the comment */\n\t\t\t\t\t\tINSERT /* inline comment */ INTO tbl VALUES ('lkjafd''lksjadlf;lk\\\\jasdflkasdf\"asdklf\\\\');\n\t\t\t\t\t\t-- this is the comment\n\t\t\t\t\t\tINSERT INTO tbl VALUES ('fasf_bkdjlfa');\n\t\t\t\t\t\t$$;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 8, column: 10}
- range: {start: 0, end: 303}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 8
+ column: 10
+ range:
+ start: 0
+ end: 303
empty: false
- text: "\n\t\t\t\t\t\tCREATE TABLE t(a int);"
baseline: 7
- start: {line: 9, column: 7}
- end: {line: 9, column: 29}
- range: {start: 303, end: 332}
- empty: false
-
-- description: "procedure with tagged dollar-quote containing nested dollar signs"
+ start:
+ line: 8
+ column: 10
+ end:
+ line: 9
+ column: 29
+ range:
+ start: 303
+ end: 332
+ empty: false
+- description: procedure with tagged dollar-quote containing nested dollar signs
input: "CREATE PROCEDURE insert_data(a varchar(50), b varchar(50))\n\t\t\t\t\t\tLANGUAGE SQL\n\t\t\t\t\t\tAS $tag_name$\n\t\t\t\t\t\t/*this is the comment */\n\t\t\t\t\t\tINSERT /* inline comment */ INTO tbl VALUES ('lkjafd''lksjadlf;lkjasdflkasdf\"asdklf');\n\t\t\t\t\t\t-- this is the comment\n\t\t\t\t\t\tINSERT INTO tbl VALUES ('fasf_bkdjlfa');\n\t\t\t\t\t\t$tag_name$;\n\t\t\t\t\t\tCREATE TABLE t(a int);"
result:
- text: "CREATE PROCEDURE insert_data(a varchar(50), b varchar(50))\n\t\t\t\t\t\tLANGUAGE SQL\n\t\t\t\t\t\tAS $tag_name$\n\t\t\t\t\t\t/*this is the comment */\n\t\t\t\t\t\tINSERT /* inline comment */ INTO tbl VALUES ('lkjafd''lksjadlf;lkjasdflkasdf\"asdklf');\n\t\t\t\t\t\t-- this is the comment\n\t\t\t\t\t\tINSERT INTO tbl VALUES ('fasf_bkdjlfa');\n\t\t\t\t\t\t$tag_name$;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 8, column: 18}
- range: {start: 0, end: 315}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 8
+ column: 18
+ range:
+ start: 0
+ end: 315
empty: false
- text: "\n\t\t\t\t\t\tCREATE TABLE t(a int);"
baseline: 7
- start: {line: 9, column: 7}
- end: {line: 9, column: 29}
- range: {start: 315, end: 344}
- empty: false
-
-- description: "multiple statements with leading whitespace and no trailing semicolon"
+ start:
+ line: 8
+ column: 18
+ end:
+ line: 9
+ column: 29
+ range:
+ start: 315
+ end: 344
+ empty: false
+- description: multiple statements with leading whitespace and no trailing semicolon
input: " CREATE TABLE t(a int); CREATE TABLE t1(a int)"
result:
- text: " CREATE TABLE t(a int);"
baseline: 0
- start: {line: 1, column: 5}
- end: {line: 1, column: 27}
- range: {start: 0, end: 26}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 27
+ range:
+ start: 0
+ end: 26
empty: false
- text: " CREATE TABLE t1(a int)"
baseline: 0
- start: {line: 1, column: 28}
- end: {line: 1, column: 50}
- range: {start: 26, end: 49}
- empty: false
-
-- description: "complex INSERT with escaped strings and special characters"
+ start:
+ line: 1
+ column: 27
+ end:
+ line: 1
+ column: 50
+ range:
+ start: 26
+ end: 49
+ empty: false
+- description: "position semantic: leading newlines"
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: "position semantic: leading spaces and newlines"
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: "position semantic: multi-statement with leading whitespace"
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: "SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
+ empty: false
+- description: complex INSERT with escaped strings and special characters
input: "INSERT INTO \"public\".\"table\"(\"id\",\"content\")\n\t\t\tVALUES\n\t\t\t(12,'table column name () { :xna,sydfn,,kasdfyn;}; /////test string/// 0'),\n\t\t\t(133,'knuandfan public table id'';create table t(a int, b int);set @text=''\\\\\\\\kdaminxkljasdfiebkla.unkonwn''+''abcdef.xyz\\\\''; local xxxyy.abcddd.mysql @text;------- '),\n\t\t\t(1444,'table t xyz abc a''a\\\\\\\\\\\\\\\\''b\"c>?>xxxxxx%}}%%>c<[[?${12344556778990{%}}cake\\\\');"
result:
- text: "INSERT INTO \"public\".\"table\"(\"id\",\"content\")\n\t\t\tVALUES\n\t\t\t(12,'table column name () { :xna,sydfn,,kasdfyn;}; /////test string/// 0'),\n\t\t\t(133,'knuandfan public table id'';create table t(a int, b int);set @text=''\\\\\\\\kdaminxkljasdfiebkla.unkonwn''+''abcdef.xyz\\\\''; local xxxyy.abcddd.mysql @text;------- '),\n\t\t\t(1444,'table t xyz abc a''a\\\\\\\\\\\\\\\\''b\"c>?>xxxxxx%}}%%>c<[[?${12344556778990{%}}cake\\\\');"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 5, column: 93}
- range: {start: 0, end: 400}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 5
+ column: 93
+ range:
+ start: 0
+ end: 400
empty: false
diff --git a/backend/plugin/parser/plsql/completion.go b/backend/plugin/parser/plsql/completion.go
index a74661dd0a..96bc115eb1 100644
--- a/backend/plugin/parser/plsql/completion.go
+++ b/backend/plugin/parser/plsql/completion.go
@@ -1171,8 +1171,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
// The caret is in the same line as the last line of the previous SQL statement.
- // We need to adjust the caret offset.
- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.
+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.
+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)
+ newCaretOffset = caretOffset - previousSQLEndColumn + 1
}
break
}
diff --git a/backend/plugin/parser/plsql/plsql.go b/backend/plugin/parser/plsql/plsql.go
index e1922732c4..27f839a21f 100644
--- a/backend/plugin/parser/plsql/plsql.go
+++ b/backend/plugin/parser/plsql/plsql.go
@@ -119,22 +119,40 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {
// Iterate through children in order to preserve statement ordering and re-parse each one
var result []*base.ANTLRAST
+ prevStopTokenIndex := -1
for _, child := range sqlScript.GetChildren() {
var stmtText string
var stmtBaseLine int
+ var startToken, stopToken antlr.Token
// Type assert to get the specific statement type
if stmt, ok := child.(parser.IUnit_statementContext); ok {
- stmtText = tokens.GetTextFromTokens(stmt.GetStart(), stmt.GetStop())
- stmtBaseLine = stmt.GetStart().GetLine() - 1 // Convert to 0-based
+ startToken = stmt.GetStart()
+ stopToken = stmt.GetStop()
} else if sqlPlusCmd, ok := child.(parser.ISql_plus_commandContext); ok {
- stmtText = tokens.GetTextFromTokens(sqlPlusCmd.GetStart(), sqlPlusCmd.GetStop())
- stmtBaseLine = sqlPlusCmd.GetStart().GetLine() - 1 // Convert to 0-based
+ startToken = sqlPlusCmd.GetStart()
+ stopToken = sqlPlusCmd.GetStop()
} else {
// Skip other node types (e.g., EOF)
continue
}
+ // Calculate the leading whitespace/comments before this statement (like SplitSQL does)
+ leadingContent := ""
+ if startTokenIndex := startToken.GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {
+ leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(startTokenIndex-1))
+ }
+
+ // Include leading whitespace in the statement text
+ stmtText = leadingContent + tokens.GetTextFromTokens(startToken, stopToken)
+
+ // stmtBaseLine is where the leading content starts (for re-parsing with correct offsets).
+ // This ensures token positions in the re-parsed AST are correct when combined with SplitSQL's BaseLine.
+ // Formula: first token's line - 1 (convert to 0-based) - number of newlines in leading content
+ stmtBaseLine = startToken.GetLine() - 1 - strings.Count(leadingContent, "\n")
+
+ prevStopTokenIndex = stopToken.GetTokenIndex()
+
// Skip empty statements
if strings.TrimSpace(stmtText) == "" || stmtText == ";" {
continue
@@ -146,6 +164,8 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {
return nil, err
}
+ // StartPosition points to the first character of Text (including leading whitespace),
+ // not the first token. This enables replacing BaseLine with Start.GetLine() - 1 in advisors.
result = append(result, &base.ANTLRAST{
StartPosition: &storepb.Position{Line: int32(stmtBaseLine) + 1},
Tree: stmtTree,
diff --git a/backend/plugin/parser/plsql/plsql_test.go b/backend/plugin/parser/plsql/plsql_test.go
index 6c17bbd58f..47695443e8 100644
--- a/backend/plugin/parser/plsql/plsql_test.go
+++ b/backend/plugin/parser/plsql/plsql_test.go
@@ -61,7 +61,7 @@ func TestPLSQLParser_MultipleStatements(t *testing.T) {
tests := []struct {
statement string
expectedCount int // Number of individual statements
- expectedLines []int // BaseLine for each result
+ expectedLines []int // StartPosition.Line - 1 for each result (0-based line of first character, including leading whitespace)
}{
{
statement: "SELECT * FROM t1;",
@@ -69,17 +69,19 @@ func TestPLSQLParser_MultipleStatements(t *testing.T) {
expectedLines: []int{0},
},
{
+ // Statement 2's first character is the newline at end of line 1
statement: `SELECT * FROM t1;
SELECT * FROM t2;`,
expectedCount: 2,
- expectedLines: []int{0, 1},
+ expectedLines: []int{0, 0},
},
{
+ // Statement 2's first char is newline at end of line 1, statement 3's first char is newline at end of line 2
statement: `SELECT * FROM t1;
SELECT * FROM t2;
INSERT INTO t3 VALUES (1, 2);`,
expectedCount: 3,
- expectedLines: []int{0, 1, 2},
+ expectedLines: []int{0, 0, 1},
},
}
diff --git a/backend/plugin/parser/plsql/restore.go b/backend/plugin/parser/plsql/restore.go
index df8997e1de..e76da66dcb 100644
--- a/backend/plugin/parser/plsql/restore.go
+++ b/backend/plugin/parser/plsql/restore.go
@@ -337,7 +337,8 @@ func extractSQL(statement string, backupItem *storepb.PriorBackupDetail_Item) (s
result = append(result, list[i].Text)
}
}
- return strings.Join(result, ";\n"), nil
+ // Statements include their leading whitespace and trailing semicolons.
+ return strings.Join(result, ""), nil
}
func equalOrLess(a, b *storepb.Position) bool {
diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.go
index f021b85b41..e6259259db 100644
--- a/backend/plugin/parser/plsql/split.go
+++ b/backend/plugin/parser/plsql/split.go
@@ -1,15 +1,12 @@
package plsql
import (
- "strings"
-
"github.com/antlr4-go/antlr/v4"
parser "github.com/bytebase/parser/plsql"
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
- "github.com/bytebase/bytebase/backend/utils"
)
func init() {
@@ -35,44 +32,66 @@ func SplitSQL(statement string) ([]base.Statement, error) {
prevStopTokenIndex := -1
var result []base.Statement
for _, item := range tree.GetChildren() {
+ // Skip past sql_plus_command (like "/" block terminator) to prevent it from being
+ // included in the next statement's leadingContent.
+ if sqlPlusCmd, ok := item.(parser.ISql_plus_commandContext); ok {
+ // Calculate the leading whitespace/comments before this sql_plus_command
+ leadingContent := ""
+ if startTokenIndex := sqlPlusCmd.GetStart().GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {
+ leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(sqlPlusCmd.GetStart().GetTokenIndex()-1))
+ }
+ // Skip past both the leading content and the command itself
+ cmdText := tokens.GetTextFromTokens(sqlPlusCmd.GetStart(), sqlPlusCmd.GetStop())
+ byteOffsetStart += len(leadingContent) + len(cmdText)
+ prevStopTokenIndex = sqlPlusCmd.GetStop().GetTokenIndex()
+ continue
+ }
+
if stmt, ok := item.(parser.IUnit_statementContext); ok {
text := ""
var lastToken antlr.Token
- // tokens looks like
+
+ // Calculate the leading whitespace/comments before this statement
+ leadingContent := ""
if startTokenIndex := stmt.GetStart().GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {
- byteOffsetStart += len(tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(stmt.GetStart().GetTokenIndex()-1)))
+ leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(stmt.GetStart().GetTokenIndex()-1))
}
- byteOffsetEnd := byteOffsetStart + len(tokens.GetTextFromTokens(stmt.GetStart(), stmt.GetStop()))
- // The go-ora driver requires semicolon for anonymous block,
- // but does not support semicolon for other statements.
+ // The go-ora driver requires semicolon for anonymous blocks/procedures/functions,
+ // but does NOT support semicolon for other statements (CREATE TABLE, SELECT, etc.).
+ stopTokenIndex := stmt.GetStop().GetTokenIndex()
if needSemicolon(stmt) {
- lastToken = tokens.Get(stmt.GetStop().GetTokenIndex())
- text = tokens.GetTextFromTokens(stmt.GetStart(), lastToken)
+ // For procedures/functions/anonymous blocks: include semicolon if present, add if missing
+ lastToken = tokens.Get(stopTokenIndex)
+ text = leadingContent + tokens.GetTextFromTokens(stmt.GetStart(), lastToken)
if lastToken.GetTokenType() != parser.PlSqlParserSEMICOLON {
text += ";"
}
} else {
- stopIndex := stmt.GetStop().GetTokenIndex()
+ // For regular statements: EXCLUDE the semicolon (go-ora doesn't support it)
if stmt.GetStop().GetTokenType() == parser.PlSqlParserSEMICOLON {
- stopIndex--
+ stopTokenIndex--
}
- lastToken = tokens.Get(stopIndex)
- text = tokens.GetTextFromTokens(stmt.GetStart(), lastToken)
- text = strings.TrimRightFunc(text, utils.IsSpaceOrSemicolon)
+ lastToken = tokens.Get(stopTokenIndex)
+ text = leadingContent + tokens.GetTextFromTokens(stmt.GetStart(), lastToken)
}
+ // Calculate byte offsets using lastToken (which includes semicolon if present)
+ // byteOffsetStart is where the previous statement ended (including any leading whitespace)
+ tokenByteOffset := byteOffsetStart + len(leadingContent)
+ byteOffsetEnd := tokenByteOffset + len(tokens.GetTextFromTokens(stmt.GetStart(), lastToken))
+
+ // Calculate start position based on byteOffsetStart (including leading whitespace)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffsetStart)
+
result = append(result, base.Statement{
Text: text,
// BaseLine is 0-based line offset of this statement in the original SQL
- BaseLine: stmt.GetStart().GetLine() - 1,
- Start: common.ConvertANTLRPositionToPosition(
- &common.ANTLRPosition{
- Line: int32(stmt.GetStart().GetLine()),
- Column: int32(stmt.GetStart().GetColumn()),
- },
- statement,
- ),
+ BaseLine: startLine,
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
End: common.ConvertANTLRTokenToExclusiveEndPosition(
int32(lastToken.GetLine()),
int32(lastToken.GetColumn()),
@@ -85,7 +104,15 @@ func SplitSQL(statement string) ([]base.Statement, error) {
},
})
byteOffsetStart = byteOffsetEnd
+ // Set prevStopTokenIndex to the last token we want to "consume" for this statement.
+ // For statements where the semicolon is a separator (not part of the statement parse tree),
+ // we need to skip past the semicolon so it's not included in the next statement's leadingContent.
prevStopTokenIndex = stmt.GetStop().GetTokenIndex()
+ if nextIdx := prevStopTokenIndex + 1; nextIdx < len(tokens.GetAllTokens()) {
+ if nextToken := tokens.Get(nextIdx); nextToken.GetTokenType() == parser.PlSqlParserSEMICOLON {
+ prevStopTokenIndex = nextIdx
+ }
+ }
}
}
return result, nil
diff --git a/backend/plugin/parser/plsql/test-data/test_restore.yaml b/backend/plugin/parser/plsql/test-data/test_restore.yaml
index d0542cffdc..cf86b096fc 100644
--- a/backend/plugin/parser/plsql/test-data/test_restore.yaml
+++ b/backend/plugin/parser/plsql/test-data/test_restore.yaml
@@ -13,12 +13,12 @@
result: |-
/*
Original SQL:
- UPDATE T_GENERATED SET b = 1 WHERE a = 1;
- UPDATE T_GENERATED SET b = 2 WHERE a = 2;
- UPDATE T_GENERATED SET b = 3 WHERE a = 3;
- UPDATE T_GENERATED SET b = 4 WHERE a = 4;
- UPDATE T_GENERATED SET b = 5 WHERE a = 5;
- UPDATE T_GENERATED SET b = 6 WHERE a = 6;
+ UPDATE T_GENERATED SET b = 1 WHERE a = 1
+ UPDATE T_GENERATED SET b = 2 WHERE a = 2
+ UPDATE T_GENERATED SET b = 3 WHERE a = 3
+ UPDATE T_GENERATED SET b = 4 WHERE a = 4
+ UPDATE T_GENERATED SET b = 5 WHERE a = 5
+ UPDATE T_GENERATED SET b = 6 WHERE a = 6
UPDATE T_GENERATED SET b = 7 WHERE a = 7
*/
MERGE INTO "DB"."T_GENERATED" t
@@ -43,12 +43,12 @@
result: |-
/*
Original SQL:
- UPDATE T_GENERATED SET a = 1 WHERE b = 1;
- UPDATE T_GENERATED SET a = 2 WHERE b = 2;
- UPDATE T_GENERATED SET a = 3 WHERE b = 3;
- UPDATE T_GENERATED SET a = 4 WHERE b = 4;
- UPDATE T_GENERATED SET a = 5 WHERE b = 5;
- UPDATE T_GENERATED SET a = 6 WHERE b = 6;
+ UPDATE T_GENERATED SET a = 1 WHERE b = 1
+ UPDATE T_GENERATED SET a = 2 WHERE b = 2
+ UPDATE T_GENERATED SET a = 3 WHERE b = 3
+ UPDATE T_GENERATED SET a = 4 WHERE b = 4
+ UPDATE T_GENERATED SET a = 5 WHERE b = 5
+ UPDATE T_GENERATED SET a = 6 WHERE b = 6
UPDATE T_GENERATED SET a = 7 WHERE b = 7
*/
MERGE INTO "DB"."T_GENERATED" t
diff --git a/backend/plugin/parser/plsql/test-data/test_split.yaml b/backend/plugin/parser/plsql/test-data/test_split.yaml
index 5131fb9e2d..673468e353 100644
--- a/backend/plugin/parser/plsql/test-data/test_split.yaml
+++ b/backend/plugin/parser/plsql/test-data/test_split.yaml
@@ -1,121 +1,297 @@
-- description: "multiple SELECT statements"
- input: "SELECT * FROM t1; SELECT * FROM t2"
+- description: multiple SELECT statements
+ input: SELECT * FROM t1; SELECT * FROM t2
result:
- - text: "SELECT * FROM t1"
+ - text: SELECT * FROM t1
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 17}
- range: {start: 0, end: 16}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 17
+ range:
+ start: 0
+ end: 16
empty: false
- - text: "SELECT * FROM t2"
+ - text: " SELECT * FROM t2"
baseline: 0
- start: {line: 1, column: 19}
- end: {line: 1, column: 35}
- range: {start: 18, end: 34}
+ start:
+ line: 1
+ column: 17
+ end:
+ line: 1
+ column: 35
+ range:
+ start: 16
+ end: 33
empty: false
-
-- description: "multiple statements with newlines"
+- description: multiple statements with newlines
input: "\n\t\t\t\tselect * from t;\n\t\t\t\tcreate table table$1 (id int)\n\t\t\t"
result:
- - text: "select * from t"
+ - text: "\n\t\t\t\tselect * from t"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 20
+ range:
+ start: 0
+ end: 20
+ empty: false
+ - text: "\n\t\t\t\tcreate table table$1 (id int)"
baseline: 1
- start: {line: 2, column: 5}
- end: {line: 2, column: 20}
- range: {start: 5, end: 20}
- empty: false
- - text: "create table table$1 (id int)"
- baseline: 2
- start: {line: 3, column: 5}
- end: {line: 3, column: 34}
- range: {start: 26, end: 56}
- empty: false
-
-- description: "procedure with forward slash separator"
- input: "CREATE OR REPLACE PROCEDURE test_proc IS\nBEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Hello');\nEND;\n/"
+ start:
+ line: 2
+ column: 20
+ end:
+ line: 3
+ column: 34
+ range:
+ start: 20
+ end: 54
+ empty: false
+- description: procedure with forward slash separator
+ input: |-
+ CREATE OR REPLACE PROCEDURE test_proc IS
+ BEGIN
+ CALL DBMS_OUTPUT.PUT_LINE('Hello');
+ END;
+ /
result:
- - text: "CREATE OR REPLACE PROCEDURE test_proc IS\nBEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Hello');\nEND;"
+ - text: |-
+ CREATE OR REPLACE PROCEDURE test_proc IS
+ BEGIN
+ CALL DBMS_OUTPUT.PUT_LINE('Hello');
+ END;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 4, column: 5}
- range: {start: 0, end: 91}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 4
+ column: 5
+ range:
+ start: 0
+ end: 91
empty: false
-
-- description: "two procedures with forward slash separator"
- input: "CREATE OR REPLACE PROCEDURE proc1 IS\nBEGIN\n NULL;\nEND;\n/\nCREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;\n/"
+- description: two procedures with forward slash separator
+ input: |-
+ CREATE OR REPLACE PROCEDURE proc1 IS
+ BEGIN
+ NULL;
+ END;
+ /
+ CREATE OR REPLACE PROCEDURE proc2 IS
+ BEGIN
+ NULL;
+ END;
+ /
result:
- - text: "CREATE OR REPLACE PROCEDURE proc1 IS\nBEGIN\n NULL;\nEND;"
+ - text: |-
+ CREATE OR REPLACE PROCEDURE proc1 IS
+ BEGIN
+ NULL;
+ END;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 4, column: 5}
- range: {start: 0, end: 57}
- empty: false
- - text: "CREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;"
- baseline: 5
- start: {line: 6, column: 1}
- end: {line: 9, column: 5}
- range: {start: 60, end: 117}
- empty: false
-
-- description: "SELECT statements separated by forward slash"
- input: "SELECT * FROM t1;\n/\nSELECT * FROM t2;"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 4
+ column: 5
+ range:
+ start: 0
+ end: 57
+ empty: false
+ - text: "\nCREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;"
+ baseline: 4
+ start:
+ line: 5
+ column: 2
+ end:
+ line: 9
+ column: 5
+ range:
+ start: 59
+ end: 117
+ empty: false
+- description: SELECT statements separated by forward slash
+ input: |-
+ SELECT * FROM t1;
+ /
+ SELECT * FROM t2;
result:
- - text: "SELECT * FROM t1"
+ - text: SELECT * FROM t1
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 17}
- range: {start: 0, end: 16}
- empty: false
- - text: "SELECT * FROM t2"
- baseline: 2
- start: {line: 3, column: 1}
- end: {line: 3, column: 17}
- range: {start: 20, end: 36}
- empty: false
-
-- description: "anonymous block with forward slash"
- input: "BEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Test');\nEND;\n/"
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 17
+ range:
+ start: 0
+ end: 16
+ empty: false
+ - text: "\nSELECT * FROM t2"
+ baseline: 1
+ start:
+ line: 2
+ column: 1
+ end:
+ line: 3
+ column: 17
+ range:
+ start: 18
+ end: 35
+ empty: false
+- description: anonymous block with forward slash
+ input: |-
+ BEGIN
+ CALL DBMS_OUTPUT.PUT_LINE('Test');
+ END;
+ /
result:
- - text: "BEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Test');\nEND;"
+ - text: |-
+ BEGIN
+ CALL DBMS_OUTPUT.PUT_LINE('Test');
+ END;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 3, column: 5}
- range: {start: 0, end: 49}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 5
+ range:
+ start: 0
+ end: 49
empty: false
-
-- description: "ALTER TABLE with PARTITION"
- input: "ALTER TABLE DATA.TEST\nMODIFY PARTITION BY RANGE (TXN_DATE)\nINTERVAL (NUMTODSINTERVAL(1, 'DAY'))\n(\n\tPARTITION TEST_PO VALUES LESS THAN (\n\t\tTO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')\n\t)\n)\nONLINE;"
+- description: ALTER TABLE with PARTITION
+ input: |-
+ ALTER TABLE DATA.TEST
+ MODIFY PARTITION BY RANGE (TXN_DATE)
+ INTERVAL (NUMTODSINTERVAL(1, 'DAY'))
+ (
+ PARTITION TEST_PO VALUES LESS THAN (
+ TO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')
+ )
+ )
+ ONLINE;
result:
- - text: "ALTER TABLE DATA.TEST\nMODIFY PARTITION BY RANGE (TXN_DATE)\nINTERVAL (NUMTODSINTERVAL(1, 'DAY'))\n(\n\tPARTITION TEST_PO VALUES LESS THAN (\n\t\tTO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')\n\t)\n)\nONLINE"
+ - text: |-
+ ALTER TABLE DATA.TEST
+ MODIFY PARTITION BY RANGE (TXN_DATE)
+ INTERVAL (NUMTODSINTERVAL(1, 'DAY'))
+ (
+ PARTITION TEST_PO VALUES LESS THAN (
+ TO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')
+ )
+ )
+ ONLINE
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 9, column: 7}
- range: {start: 0, end: 233}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 9
+ column: 7
+ range:
+ start: 0
+ end: 232
empty: false
-
-- description: "CALL procedure"
- input: "CALL my_procedure()"
+- description: CALL procedure
+ input: CALL my_procedure()
result:
- - text: "CALL my_procedure()"
+ - text: CALL my_procedure()
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 20}
- range: {start: 0, end: 19}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 20
+ range:
+ start: 0
+ end: 19
empty: false
-
-- description: "DROP TABLESPACE with CASCADE CONSTRAINTS"
- input: "DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS"
+- description: DROP TABLESPACE with CASCADE CONSTRAINTS
+ input: DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS
result:
- - text: "DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS"
+ - text: DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 59}
- range: {start: 0, end: 58}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 59
+ range:
+ start: 0
+ end: 58
empty: false
-
-- description: "DROP TABLESPACE CASCADE alone should error"
- input: "DROP TABLESPACE xxx CASCADE"
+- description: DROP TABLESPACE CASCADE alone should error
+ input: DROP TABLESPACE xxx CASCADE
error: "Syntax error at line 1:28 \nrelated text: DROP TABLESPACE xxx CASCADE;"
-
-- description: "DROP TABLESPACE then CASCADE should error"
- input: "DROP TABLESPACE xxx; CASCADE"
+- description: DROP TABLESPACE then CASCADE should error
+ input: DROP TABLESPACE xxx; CASCADE
error: "Syntax error at line 1:29 \nrelated text: DROP TABLESPACE xxx; CASCADE;"
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1 FROM dual"
+ result:
+ - text: "\n\nSELECT 1 FROM dual"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 19
+ range:
+ start: 0
+ end: 20
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1 FROM dual"
+ result:
+ - text: "\n \n SELECT 1 FROM dual"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 21
+ range:
+ start: 0
+ end: 24
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1 FROM dual;\n\n SELECT 2 FROM dual"
+ result:
+ - text: SELECT 1 FROM dual
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 19
+ range:
+ start: 0
+ end: 18
+ empty: false
+ - text: "\n\n SELECT 2 FROM dual"
+ baseline: 0
+ start:
+ line: 1
+ column: 19
+ end:
+ line: 3
+ column: 21
+ range:
+ start: 18
+ end: 40
+ empty: false
diff --git a/backend/plugin/parser/redshift/completion.go b/backend/plugin/parser/redshift/completion.go
index e92097e1f1..e40f18921c 100644
--- a/backend/plugin/parser/redshift/completion.go
+++ b/backend/plugin/parser/redshift/completion.go
@@ -1287,8 +1287,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
// The caret is in the same line as the last line of the previous SQL statement.
- // We need to adjust the caret offset.
- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.
+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.
+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)
+ newCaretOffset = caretOffset - previousSQLEndColumn + 1
}
break
}
diff --git a/backend/plugin/parser/redshift/split.go b/backend/plugin/parser/redshift/split.go
index d16d3ecbde..0c1fb35bfc 100644
--- a/backend/plugin/parser/redshift/split.go
+++ b/backend/plugin/parser/redshift/split.go
@@ -71,9 +71,8 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -86,7 +85,10 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:pos+1], parser.RedshiftParserSEMI),
})
byteOffset += stmtByteLength
@@ -98,9 +100,8 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.
stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -113,7 +114,10 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.
int32(tokens[eofPos-1].GetColumn()),
tokens[eofPos-1].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:eofPos], parser.RedshiftParserSEMI),
})
}
@@ -207,9 +211,8 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -222,7 +225,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:pos+1], parser.RedshiftParserSEMI),
})
byteOffset += stmtByteLength
@@ -234,9 +240,8 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
- // From antlr4, the line is ONE based, and the column is ZERO based.
- // So we should minus 1 for the line.
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -249,7 +254,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
int32(tokens[eofPos-1].GetColumn()),
tokens[eofPos-1].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:eofPos], parser.RedshiftParserSEMI),
})
}
diff --git a/backend/plugin/parser/redshift/test-data/test_split.yaml b/backend/plugin/parser/redshift/test-data/test_split.yaml
index e2ef44caae..8ac26bfca8 100644
--- a/backend/plugin/parser/redshift/test-data/test_split.yaml
+++ b/backend/plugin/parser/redshift/test-data/test_split.yaml
@@ -1,51 +1,141 @@
-- description: "simple single statement"
- input: "SELECT 1;"
+- description: simple single statement
+ input: SELECT 1;
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
-
-- description: "multiple statements"
- input: "SELECT 1; SELECT 2;"
+- description: multiple statements
+ input: SELECT 1; SELECT 2;
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- - text: " SELECT 2;"
+ - text: ' SELECT 2;'
baseline: 0
- start: {line: 1, column: 11}
- end: {line: 1, column: 20}
- range: {start: 9, end: 19}
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 1
+ column: 20
+ range:
+ start: 9
+ end: 19
empty: false
-
-- description: "multiple statements on multiple lines"
+- description: multiple statements on multiple lines
input: "SELECT 1;\nSELECT 2;"
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- text: "\nSELECT 2;"
baseline: 0
- start: {line: 2, column: 1}
- end: {line: 2, column: 10}
- range: {start: 9, end: 19}
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 2
+ column: 10
+ range:
+ start: 9
+ end: 19
empty: false
-
-- description: "multi-line statement"
+- description: multi-line statement
input: "SELECT\n 1;"
result:
- text: "SELECT\n 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 5}
- range: {start: 0, end: 11}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 5
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/snowflake/split.go b/backend/plugin/parser/snowflake/split.go
index 88e01e96f7..5fbb47de13 100644
--- a/backend/plugin/parser/snowflake/split.go
+++ b/backend/plugin/parser/snowflake/split.go
@@ -18,5 +18,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {
stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)
stream.Fill()
- return base.SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI, statement)
+ return base.SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI)
}
diff --git a/backend/plugin/parser/snowflake/test-data/test_split.yaml b/backend/plugin/parser/snowflake/test-data/test_split.yaml
index 195f6fee52..c68bc213de 100644
--- a/backend/plugin/parser/snowflake/test-data/test_split.yaml
+++ b/backend/plugin/parser/snowflake/test-data/test_split.yaml
@@ -1,41 +1,126 @@
-- description: "multiple statements on multiple lines"
+- description: multiple statements on multiple lines
input: "SELECT * FROM users;\n\tSELECT * FROM orders;\n\tSELECT * FROM products;"
result:
- - text: "SELECT * FROM users;"
+ - text: SELECT * FROM users;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 21}
- range: {start: 0, end: 20}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 21
+ range:
+ start: 0
+ end: 20
empty: false
- text: "\n\tSELECT * FROM orders;"
baseline: 0
- start: {line: 2, column: 2}
- end: {line: 2, column: 23}
- range: {start: 20, end: 43}
+ start:
+ line: 1
+ column: 21
+ end:
+ line: 2
+ column: 23
+ range:
+ start: 20
+ end: 43
empty: false
- text: "\n\tSELECT * FROM products;"
baseline: 1
- start: {line: 3, column: 2}
- end: {line: 3, column: 25}
- range: {start: 43, end: 68}
+ start:
+ line: 2
+ column: 23
+ end:
+ line: 3
+ column: 25
+ range:
+ start: 43
+ end: 68
empty: false
-
-- description: "single statement"
- input: "SELECT * FROM users;"
+- description: single statement
+ input: SELECT * FROM users;
result:
- - text: "SELECT * FROM users;"
+ - text: SELECT * FROM users;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 21}
- range: {start: 0, end: 20}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 21
+ range:
+ start: 0
+ end: 20
empty: false
-
-- description: "statement without semicolon"
- input: "SELECT * FROM users"
+- description: statement without semicolon
+ input: SELECT * FROM users
result:
- - text: "SELECT * FROM users"
+ - text: SELECT * FROM users
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 20}
- range: {start: 0, end: 19}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 20
+ range:
+ start: 0
+ end: 19
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/spanner/split.go b/backend/plugin/parser/spanner/split.go
index a01347d955..c600d6582d 100644
--- a/backend/plugin/parser/spanner/split.go
+++ b/backend/plugin/parser/spanner/split.go
@@ -92,7 +92,8 @@ func SplitSQL(statement string) ([]base.Statement, error) {
stmtText := stream.GetTextFromTokens(tokens[start], tokens[endPos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : endPos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -105,7 +106,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {
int32(tokens[endPos].GetColumn()),
tokens[endPos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: base.IsEmpty(tokens[start:endPos+1], parser.GoogleSQLLexerSEMI_SYMBOL),
})
byteOffset += stmtByteLength
diff --git a/backend/plugin/parser/spanner/test-data/test_split.yaml b/backend/plugin/parser/spanner/test-data/test_split.yaml
index 0043d41600..60ded7187e 100644
--- a/backend/plugin/parser/spanner/test-data/test_split.yaml
+++ b/backend/plugin/parser/spanner/test-data/test_split.yaml
@@ -1,31 +1,111 @@
-- description: "multiple statements on multiple lines"
+- description: multiple statements on multiple lines
input: "SELECT * FROM users;\n\tSELECT * FROM orders;\n\tSELECT * FROM products;"
result:
- - text: "SELECT * FROM users;"
+ - text: SELECT * FROM users;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 21}
- range: {start: 0, end: 20}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 21
+ range:
+ start: 0
+ end: 20
empty: false
- text: "\n\tSELECT * FROM orders;"
baseline: 0
- start: {line: 2, column: 2}
- end: {line: 2, column: 23}
- range: {start: 20, end: 43}
+ start:
+ line: 1
+ column: 21
+ end:
+ line: 2
+ column: 23
+ range:
+ start: 20
+ end: 43
empty: false
- text: "\n\tSELECT * FROM products;"
baseline: 1
- start: {line: 3, column: 2}
- end: {line: 3, column: 25}
- range: {start: 43, end: 68}
+ start:
+ line: 2
+ column: 23
+ end:
+ line: 3
+ column: 25
+ range:
+ start: 43
+ end: 68
empty: false
-
-- description: "single statement"
- input: "SELECT * FROM users;"
+- description: single statement
+ input: SELECT * FROM users;
result:
- - text: "SELECT * FROM users;"
+ - text: SELECT * FROM users;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 21}
- range: {start: 0, end: 20}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 21
+ range:
+ start: 0
+ end: 20
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/standard/split.go b/backend/plugin/parser/standard/split.go
index 43a76644e4..8313bb2cec 100644
--- a/backend/plugin/parser/standard/split.go
+++ b/backend/plugin/parser/standard/split.go
@@ -24,22 +24,31 @@ func SplitSQL(statement string) ([]base.Statement, error) {
var list []base.Statement
byteOffset := 0
err := applyMultiStatements(strings.NewReader(statement), func(sql string) error {
- // Find the start position of this SQL in the original statement
- startPos := strings.Index(statement[byteOffset:], strings.TrimLeft(sql, "\n\t "))
- if startPos != -1 {
- startPos += byteOffset
- } else {
- startPos = byteOffset
+ // sql may have trailing newline added by applyMultiStatements
+ // Strip trailing whitespace for searching in the original statement
+ sqlTrimmed := strings.TrimRight(sql, "\n\r\t ")
+
+ // Find where the SQL content exists in the remaining statement
+ sqlPos := strings.Index(statement[byteOffset:], sqlTrimmed)
+ if sqlPos == -1 {
+ sqlPos = 0
}
- endPos := startPos + len(sql)
+
+ // startPos includes leading whitespace from where previous statement ended
+ startPos := byteOffset
+ // endPos is where the SQL content ends
+ endPos := byteOffset + sqlPos + len(sqlTrimmed)
+
+ // The actual text includes leading whitespace + SQL content
+ text := statement[startPos:endPos]
// Calculate line and column for Start position
- startLine, startColumn := calculateLineAndColumn(statement, startPos)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, startPos)
// Calculate line and column for End position
- endLine, endColumn := calculateLineAndColumn(statement, endPos)
+ endLine, endColumn := base.CalculateLineAndColumn(statement, endPos)
list = append(list, base.Statement{
- Text: sql,
+ Text: text,
BaseLine: startLine,
Start: &storepb.Position{
Line: int32(startLine + 1), // 1-based
@@ -53,7 +62,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {
Start: int32(startPos),
End: int32(endPos),
},
- Empty: isEmptySQL(sql),
+ Empty: isEmptySQL(text),
})
byteOffset = endPos
return nil
@@ -61,24 +70,6 @@ func SplitSQL(statement string) ([]base.Statement, error) {
return list, err
}
-// calculateLineAndColumn calculates the 0-based line number and 0-based column (character offset)
-// for a given byte offset in the statement.
-func calculateLineAndColumn(statement string, byteOffset int) (line, column int) {
- if byteOffset > len(statement) {
- byteOffset = len(statement)
- }
- // Range over string iterates over runes (code points), not bytes
- for _, r := range statement[:byteOffset] {
- if r == '\n' {
- line++
- column = 0
- } else {
- column++
- }
- }
- return line, column
-}
-
// isEmptySQL checks if the SQL contains only whitespace and comments.
func isEmptySQL(sql string) bool {
trimmed := strings.TrimSpace(sql)
@@ -173,7 +164,7 @@ func applyMultiStatements(sc io.Reader, f func(string) error) error {
}
if execute {
s := sb.String()
- s = strings.Trim(s, "\n\t ")
+ // Don't trim - include leading whitespace in Text for position consistency
if s != "" {
if err := f(s); err != nil {
return errors.Wrapf(err, "execute query %q failed", s)
@@ -184,7 +175,7 @@ func applyMultiStatements(sc io.Reader, f func(string) error) error {
}
// Apply the remaining content.
s := sb.String()
- s = strings.Trim(s, "\n\t ")
+ // Don't trim - include leading whitespace in Text for position consistency
if s != "" {
if err := f(s); err != nil {
return errors.Wrapf(err, "execute query %q failed", s)
diff --git a/backend/plugin/parser/standard/test-data/test_split.yaml b/backend/plugin/parser/standard/test-data/test_split.yaml
index 68a57e60d4..ed4a61600b 100644
--- a/backend/plugin/parser/standard/test-data/test_split.yaml
+++ b/backend/plugin/parser/standard/test-data/test_split.yaml
@@ -1,81 +1,186 @@
-- description: "simple single statement"
+- description: simple single statement
input: "SELECT 1;"
result:
- text: "SELECT 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
-
-- description: "multi-line statement"
+- description: multi-line statement
input: "SELECT\n 1;"
result:
- text: "SELECT\n 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 5}
- range: {start: 0, end: 11}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 5
+ range:
+ start: 0
+ end: 11
empty: false
-
-- description: "multiple statements"
+- description: multiple statements
input: "SELECT 1;\nSELECT 2;"
result:
- text: "SELECT 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- - text: "SELECT 2;"
- baseline: 1
- start: {line: 2, column: 1}
- end: {line: 2, column: 10}
- range: {start: 10, end: 19}
+ - text: "\nSELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 2
+ column: 10
+ range:
+ start: 9
+ end: 19
+ empty: false
+- description: multi-byte characters - Chinese
+ input: "SELECT '\u4E2D\u6587';"
+ result:
+ - text: "SELECT '\u4E2D\u6587';"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 13
+ range:
+ start: 0
+ end: 16
+ empty: false
+- description: multi-byte characters - emoji
+ input: "SELECT '\U0001F389';"
+ result:
+ - text: "SELECT '\U0001F389';"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 12
+ range:
+ start: 0
+ end: 14
empty: false
-
-- description: "multi-byte characters - Chinese"
- input: "SELECT '中文';"
+- description: multi-byte on second line
+ input: "SELECT\n '\u4E2D\u6587';"
result:
- - text: "SELECT '中文';"
+ - text: "SELECT\n '\u4E2D\u6587';"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 13}
- range: {start: 0, end: 16}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 8
+ range:
+ start: 0
+ end: 18
empty: false
-
-- description: "multi-byte characters - emoji"
- input: "SELECT '🎉';"
+- description: multiple statements with multi-byte on separate lines
+ input: "SELECT '\u4E2D';\nSELECT '\u6587';"
result:
- - text: "SELECT '🎉';"
+ - text: "SELECT '\u4E2D';"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 12}
- range: {start: 0, end: 14}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 12
+ range:
+ start: 0
+ end: 13
empty: false
-
-- description: "multi-byte on second line"
- input: "SELECT\n '中文';"
+ - text: "\nSELECT '\u6587';"
+ baseline: 0
+ start:
+ line: 1
+ column: 12
+ end:
+ line: 2
+ column: 12
+ range:
+ start: 13
+ end: 27
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
result:
- - text: "SELECT\n '中文';"
+ - text: "\n\nSELECT 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 8}
- range: {start: 0, end: 18}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
empty: false
-
-- description: "multiple statements with multi-byte on separate lines"
- input: "SELECT '中';\nSELECT '文';"
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
result:
- - text: "SELECT '中';"
+ - text: "\n \n SELECT 1;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 12}
- range: {start: 0, end: 13}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
empty: false
- - text: "SELECT '文';"
- baseline: 1
- start: {line: 2, column: 1}
- end: {line: 2, column: 12}
- range: {start: 14, end: 27}
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: "SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/tidb/test-data/test_backup.yaml b/backend/plugin/parser/tidb/test-data/test_backup.yaml
index a0af5507c4..7576cbd1ad 100644
--- a/backend/plugin/parser/tidb/test-data/test_backup.yaml
+++ b/backend/plugin/parser/tidb/test-data/test_backup.yaml
@@ -54,8 +54,8 @@
sourcetablename: t2
targettablename: _rollback_1_t2
startposition:
- line: 2
- column: 1
+ line: 1
+ column: 39
endposition:
line: 2
column: 39
@@ -244,8 +244,8 @@
sourcetablename: test
targettablename: _rollback_1_test
startposition:
- line: 2
- column: 1
+ line: 1
+ column: 47
endposition:
line: 2
column: 47
diff --git a/backend/plugin/parser/tidb/test-data/test_split.yaml b/backend/plugin/parser/tidb/test-data/test_split.yaml
index 18ce18cd61..f7aad5bbba 100644
--- a/backend/plugin/parser/tidb/test-data/test_split.yaml
+++ b/backend/plugin/parser/tidb/test-data/test_split.yaml
@@ -1,91 +1,209 @@
-- description: "simple single statement"
- input: "SELECT 1;"
+- description: simple single statement
+ input: SELECT 1;
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
-
-- description: "multi-line statement"
- input: "SELECT\n 1;"
+- description: multi-line statement
+ input: |-
+ SELECT
+ 1;
result:
- - text: "SELECT\n 1;"
+ - text: |-
+ SELECT
+ 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 5}
- range: {start: 0, end: 11}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 5
+ range:
+ start: 0
+ end: 11
empty: false
-
-- description: "multiple statements"
+- description: multiple statements
input: "SELECT 1;\nSELECT 2;"
result:
- - text: "SELECT 1;"
+ - text: SELECT 1;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 10}
- range: {start: 0, end: 9}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
empty: false
- - text: "SELECT 2;"
- baseline: 1
- start: {line: 2, column: 1}
- end: {line: 2, column: 10}
- range: {start: 10, end: 19}
+ - text: "\nSELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 2
+ column: 10
+ range:
+ start: 9
+ end: 19
empty: false
-
-- description: "multi-byte characters - Chinese"
- input: "SELECT '中文';"
+- description: multi-byte characters - Chinese
+ input: SELECT '中文';
result:
- - text: "SELECT '中文';"
+ - text: SELECT '中文';
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 13}
- range: {start: 0, end: 16}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 13
+ range:
+ start: 0
+ end: 16
empty: false
-
-- description: "multi-byte characters - emoji"
- input: "SELECT '🎉';"
+- description: multi-byte characters - emoji
+ input: "SELECT '\U0001F389';"
result:
- - text: "SELECT '🎉';"
+ - text: "SELECT '\U0001F389';"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 12}
- range: {start: 0, end: 14}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 12
+ range:
+ start: 0
+ end: 14
empty: false
-
-- description: "multi-byte on second line"
- input: "SELECT\n '中文';"
+- description: multi-byte on second line
+ input: |-
+ SELECT
+ '中文';
result:
- - text: "SELECT\n '中文';"
+ - text: |-
+ SELECT
+ '中文';
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 2, column: 8}
- range: {start: 0, end: 18}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 8
+ range:
+ start: 0
+ end: 18
empty: false
-
-- description: "multiple statements with multi-byte"
- input: "SELECT '中';\nSELECT '文';"
+- description: multiple statements with multi-byte
+ input: "SELECT '\u4E2D';\nSELECT '\u6587';"
result:
- - text: "SELECT '中';"
+ - text: "SELECT '\u4E2D';"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 12}
- range: {start: 0, end: 13}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 12
+ range:
+ start: 0
+ end: 13
empty: false
- - text: "SELECT '文';"
- baseline: 1
- start: {line: 2, column: 1}
- end: {line: 2, column: 12}
- range: {start: 14, end: 27}
+ - text: "\nSELECT '\u6587';"
+ baseline: 0
+ start:
+ line: 1
+ column: 12
+ end:
+ line: 2
+ column: 12
+ range:
+ start: 13
+ end: 27
empty: false
-
-- description: "statement with leading spaces and multi-byte"
- input: " SELECT '中';"
+- description: statement with leading spaces and multi-byte
+ input: ' SELECT ''中'';'
result:
- - text: "SELECT '中';"
+ - text: ' SELECT ''中'';'
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 14
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
baseline: 0
- start: {line: 1, column: 3}
- end: {line: 1, column: 14}
- range: {start: 2, end: 15}
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/tidb/tidb.go b/backend/plugin/parser/tidb/tidb.go
index 450ee80a00..8cb9a2574f 100644
--- a/backend/plugin/parser/tidb/tidb.go
+++ b/backend/plugin/parser/tidb/tidb.go
@@ -57,15 +57,25 @@ func ParseTiDBForSyntaxCheck(statement string) ([]base.AST, error) {
}
node := nodes[0]
- node.SetText(nil, singleSQL.Text)
- node.SetOriginTextPosition(int(singleSQL.End.GetLine()))
+ // node.Text() includes leading whitespace from singleSQL.Text.
+ // This maintains consistency: Statement.Start points to first char of Statement.Text,
+ // and AST position matches Statement position.
+ // Trim only at display points (e.g., error messages) where needed.
+
+ // Calculate the start line. The native TiDB parser may strip leading newlines
+ // from its internal Text(), so we count how many were stripped.
+ nativeText := node.Text()
+ leadingNewlinesStripped := strings.Count(singleSQL.Text, "\n") - strings.Count(nativeText, "\n")
+ actualStartLine := singleSQL.BaseLine + leadingNewlinesStripped + 1
+
+ node.SetOriginTextPosition(actualStartLine)
if n, ok := node.(*ast.CreateTableStmt); ok {
if err := SetLineForMySQLCreateTableStmt(n); err != nil {
- return nil, errors.Wrapf(err, "failed to set line for create table statement at line %d", singleSQL.End.GetLine())
+ return nil, errors.Wrapf(err, "failed to set line for create table statement at line %d", actualStartLine)
}
}
results = append(results, &AST{
- StartPosition: &storepb.Position{Line: int32(singleSQL.BaseLine) + 1},
+ StartPosition: &storepb.Position{Line: int32(actualStartLine)},
Node: node,
})
}
@@ -172,7 +182,9 @@ func SetLineForMySQLCreateTableStmt(node *ast.CreateTableStmt) error {
if len(node.Cols) == 0 {
return nil
}
- firstLine := node.OriginTextPosition() - strings.Count(node.Text(), "\n")
+ // OriginTextPosition() now stores the first line of the statement (1-based),
+ // so we can use it directly as firstLine.
+ firstLine := node.OriginTextPosition()
return tokenizer.NewTokenizer(node.Text()).SetLineForMySQLCreateTableStmt(node, firstLine)
}
diff --git a/backend/plugin/parser/tidb/tidb_test.go b/backend/plugin/parser/tidb/tidb_test.go
index 64f2348a40..78f39ef5bc 100644
--- a/backend/plugin/parser/tidb/tidb_test.go
+++ b/backend/plugin/parser/tidb/tidb_test.go
@@ -1,7 +1,6 @@
package tidb
import (
- "strings"
"testing"
tidbparser "github.com/pingcap/tidb/pkg/parser"
@@ -12,7 +11,11 @@ import (
)
type setLineTestData struct {
- statement string
+ statement string
+ // firstLine is the 1-based line number where the CREATE TABLE statement starts.
+ // This is used to set OriginTextPosition, which the tokenizer uses to calculate
+ // absolute line numbers for columns and constraints.
+ firstLine int
columnLineList []int
constraintLineList []int
}
@@ -21,11 +24,13 @@ func TestMySQLCreateTableSetLine(t *testing.T) {
tests := []setLineTestData{
{
statement: "CREATE TABLE t as select * from t1",
+ firstLine: 1,
columnLineList: []int{},
constraintLineList: []int{},
},
{
statement: "CREATE TABLE t like t1",
+ firstLine: 1,
columnLineList: []int{},
constraintLineList: []int{},
},
@@ -43,12 +48,13 @@ func TestMySQLCreateTableSetLine(t *testing.T) {
-- it's a comment.
FOREIGN KEY (a, b, c) REFERENCES t1(a, b, c)
-
+
)
`,
+ firstLine: 1,
columnLineList: []int{3, 3, 4, 5},
constraintLineList: []int{6, 7, 8, 9, 10, 10, 13},
},
@@ -65,6 +71,7 @@ func TestMySQLCreateTableSetLine(t *testing.T) {
"\r\n" +
"FOREIGN KEY (a, b, c) REFERENCES t1(a, b, c)" + "\r\n" +
")",
+ firstLine: 1,
columnLineList: []int{3, 3, 4, 5},
constraintLineList: []int{6, 7, 8, 8, 10},
},
@@ -75,6 +82,7 @@ func TestMySQLCreateTableSetLine(t *testing.T) {
b int CHECK(b>1), c int UNIQUE
)
`,
+ firstLine: 1,
columnLineList: []int{3, 4, 4},
constraintLineList: []int{},
},
@@ -87,6 +95,7 @@ func TestMySQLCreateTableSetLine(t *testing.T) {
UNIQUE(name)
)
`,
+ firstLine: 1,
columnLineList: []int{3, 4},
constraintLineList: []int{5, 6},
},
@@ -102,7 +111,7 @@ func TestMySQLCreateTableSetLine(t *testing.T) {
require.True(t, ok)
require.Equal(t, len(test.columnLineList), len(node.Cols))
require.Equal(t, len(test.constraintLineList), len(node.Constraints))
- node.SetOriginTextPosition(strings.Count(test.statement, "\n") + 1)
+ node.SetOriginTextPosition(test.firstLine)
err = SetLineForMySQLCreateTableStmt(node)
require.NoError(t, err)
for i, col := range node.Cols {
diff --git a/backend/plugin/parser/tokenizer/tokenizer.go b/backend/plugin/parser/tokenizer/tokenizer.go
index a753deac3b..65afb6fb72 100644
--- a/backend/plugin/parser/tokenizer/tokenizer.go
+++ b/backend/plugin/parser/tokenizer/tokenizer.go
@@ -218,11 +218,12 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
var res []base.Statement
delimiter := []rune{';'}
- t.skipBlank()
t.emptyStatement = true
+ // Record position BEFORE skipping whitespace to include leading whitespace in Text
startPos := t.cursor
startLine := t.line // Track the starting line number (1-based)
startColumn := t.getColumn() // Track the starting column (1-based)
+ t.skipBlank()
for {
switch {
case t.char(0) == eofRune:
@@ -289,13 +290,14 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
},
})
}
- t.skipBlank()
if err := t.processStreaming(text); err != nil {
return nil, err
}
+ // Record position BEFORE skipping whitespace to include leading whitespace in Text
startPos = t.pos()
startLine = t.line // Update startLine for next statement
startColumn = t.getColumn() // Update startColumn for next statement
+ t.skipBlank()
t.emptyStatement = true
// deal with the DELIMITER statement, see https://dev.mysql.com/doc/refman/8.0/en/stored-programs-defining.html
case t.equalWordCaseInsensitive(delimiterRuneList):
@@ -325,13 +327,14 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
},
})
}
- t.skipBlank()
if err := t.processStreaming(text); err != nil {
return nil, err
}
+ // Record position BEFORE skipping whitespace to include leading whitespace in Text
startPos = t.pos()
startLine = t.line // Update startLine for next statement
startColumn = t.getColumn() // Update startColumn for next statement
+ t.skipBlank()
t.emptyStatement = true
case t.char(0) == '/' && t.char(1) == '*':
if err := t.scanComment(); err != nil {
@@ -384,11 +387,12 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {
var res []base.Statement
- t.skipBlank()
t.emptyStatement = true
+ // Record position BEFORE skipping whitespace to include leading whitespace in Text
startPos := t.cursor
firstStatementLine := t.line
firstStatementColumn := t.getColumn()
+ t.skipBlank()
for {
switch {
case t.char(0) == '/' && t.char(1) == '*':
@@ -433,13 +437,14 @@ func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {
},
})
}
- t.skipBlank()
if err := t.processStreaming(text); err != nil {
return nil, err
}
+ // Record position BEFORE skipping whitespace to include leading whitespace in Text
startPos = t.pos()
firstStatementLine = t.line
firstStatementColumn = t.getColumn()
+ t.skipBlank()
t.emptyStatement = true
case t.char(0) == eofRune:
s := t.getString(startPos, t.pos())
diff --git a/backend/plugin/parser/trino/completion.go b/backend/plugin/parser/trino/completion.go
index 8ac4831a10..021ec58b3e 100644
--- a/backend/plugin/parser/trino/completion.go
+++ b/backend/plugin/parser/trino/completion.go
@@ -1145,8 +1145,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
// The caret is in the same line as the last line of the previous SQL statement.
- // We need to adjust the caret offset.
- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.
+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.
+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)
+ newCaretOffset = caretOffset - previousSQLEndColumn + 1
}
break
}
diff --git a/backend/plugin/parser/trino/split.go b/backend/plugin/parser/trino/split.go
index 182a79d3d8..520d6c2515 100644
--- a/backend/plugin/parser/trino/split.go
+++ b/backend/plugin/parser/trino/split.go
@@ -61,6 +61,7 @@ func splitByParser(statement string) ([]base.Statement, error) {
var result []base.Statement
tokens := stream.GetAllTokens()
+ byteOffsetStart := 0
// Walk through all statements
for _, stmts := range tree.AllStatements() {
@@ -81,55 +82,37 @@ func splitByParser(statement string) ([]base.Statement, error) {
}
// Find the actual start position
- startIdx := startToken.GetTokenIndex()
endIdx := stopToken.GetTokenIndex()
- // Get the first token (any channel) for baseLine calculation
- // This is important because when ANTLR reparses fragments, it sees ALL tokens
- // including comments on hidden channels, so baseLine must be from the first token
- // of any channel for correct error position mapping: originalLine = antlrLine + baseLine
- firstToken := tokens[startIdx]
-
- // Find the first non-hidden token for accurate start position display
- var firstDefaultToken antlr.Token
- for i := startIdx; i <= endIdx && i < len(tokens); i++ {
- if tokens[i].GetChannel() == antlr.TokenDefaultChannel {
- firstDefaultToken = tokens[i]
- break
- }
- }
-
- if firstDefaultToken == nil {
- firstDefaultToken = startToken
- }
-
// Check if there's a semicolon after the statement and include it
finalEndIdx := endIdx
if endIdx+1 < len(tokens) && tokens[endIdx+1].GetTokenType() == trinoparser.TrinoLexerSEMICOLON_ {
finalEndIdx = endIdx + 1
}
- // Get the text including any trailing semicolon
- text := stream.GetTextFromInterval(antlr.NewInterval(startIdx, finalEndIdx))
-
// Calculate proper end position (1-based exclusive per proto spec)
endToken := tokens[finalEndIdx]
- // Use actual token positions for Range instead of cumulative offset
- // GetStart() returns byte offset of first character, GetStop() returns byte offset of last character
- rangeStart := firstDefaultToken.GetStart()
+ // Calculate byte range: include leading whitespace from where previous statement ended
rangeEnd := endToken.GetStop() + 1 // exclusive end
+ // Include leading whitespace in text by getting from original statement
+ // This ensures Start.Line - 1 == BaseLine for proper position mapping
+ text := statement[byteOffsetStart:rangeEnd]
+
+ // Calculate start position based on byteOffsetStart (including leading whitespace)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffsetStart)
+
result = append(result, base.Statement{
Text: text,
- BaseLine: firstToken.GetLine() - 1,
+ BaseLine: startLine,
Range: &storepb.Range{
- Start: int32(rangeStart),
+ Start: int32(byteOffsetStart),
End: int32(rangeEnd),
},
Start: &storepb.Position{
- Line: int32(firstDefaultToken.GetLine()), // 1-based (ANTLR line is already 1-based)
- Column: int32(firstDefaultToken.GetColumn() + 1), // 1-based (ANTLR column is 0-based)
+ Line: int32(startLine + 1), // 1-based
+ Column: int32(startColumn + 1), // 1-based
},
End: &storepb.Position{
Line: int32(endToken.GetLine()), // 1-based
@@ -137,6 +120,8 @@ func splitByParser(statement string) ([]base.Statement, error) {
},
Empty: false,
})
+
+ byteOffsetStart = rangeEnd
}
return result, nil
diff --git a/backend/plugin/parser/trino/test-data/test_split.yaml b/backend/plugin/parser/trino/test-data/test_split.yaml
index f0c2ee8ea9..94742a160f 100644
--- a/backend/plugin/parser/trino/test-data/test_split.yaml
+++ b/backend/plugin/parser/trino/test-data/test_split.yaml
@@ -1,31 +1,111 @@
-- description: "multiple statements same line"
- input: "SELECT * FROM users; SELECT * FROM orders;"
+- description: multiple statements same line
+ input: SELECT * FROM users; SELECT * FROM orders;
result:
- - text: "SELECT * FROM users;"
+ - text: SELECT * FROM users;
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 21}
- range: {start: 0, end: 20}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 21
+ range:
+ start: 0
+ end: 20
empty: false
- - text: "SELECT * FROM orders;"
+ - text: " SELECT * FROM orders;"
baseline: 0
- start: {line: 1, column: 22}
- end: {line: 1, column: 43}
- range: {start: 21, end: 42}
+ start:
+ line: 1
+ column: 21
+ end:
+ line: 1
+ column: 43
+ range:
+ start: 20
+ end: 42
empty: false
-
-- description: "unicode in table names"
- input: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);"
+- description: unicode in table names
+ input: "SELECT * FROM \u8868\u540D; INSERT INTO \u8868 VALUES (1);"
result:
- - text: "SELECT * FROM 表名;"
+ - text: "SELECT * FROM \u8868\u540D;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 18}
- range: {start: 0, end: 21}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 18
+ range:
+ start: 0
+ end: 21
empty: false
- - text: "INSERT INTO 表 VALUES (1);"
+ - text: " INSERT INTO \u8868 VALUES (1);"
baseline: 0
- start: {line: 1, column: 19}
- end: {line: 1, column: 44}
- range: {start: 22, end: 49}
+ start:
+ line: 1
+ column: 18
+ end:
+ line: 1
+ column: 44
+ range:
+ start: 21
+ end: 49
+ empty: false
+- description: 'position semantic: leading newlines'
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: 'position semantic: leading spaces and newlines'
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: 'position semantic: multi-statement with leading whitespace'
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: SELECT 1;
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/parser/tsql/completion.go b/backend/plugin/parser/tsql/completion.go
index d10de7bfff..5cdc2d5be6 100644
--- a/backend/plugin/parser/tsql/completion.go
+++ b/backend/plugin/parser/tsql/completion.go
@@ -1320,8 +1320,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,
newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.
if caretLine == previousSQLEndLine {
// The caret is in the same line as the last line of the previous SQL statement.
- // We need to adjust the caret offset.
- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.
+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.
+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)
+ newCaretOffset = caretOffset - previousSQLEndColumn + 1
}
break
}
diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.go
index 4432fa11e2..85322239bf 100644
--- a/backend/plugin/parser/tsql/split.go
+++ b/backend/plugin/parser/tsql/split.go
@@ -75,7 +75,8 @@ func splitByParser(statement string) ([]base.Statement, error) {
pos := goStmt.GetStop().GetTokenIndex()
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -88,7 +89,10 @@ func splitByParser(statement string) ([]base.Statement, error) {
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: false,
})
byteOffset += stmtByteLength
@@ -122,7 +126,8 @@ func splitByParser(statement string) ([]base.Statement, error) {
pos := goStmt.GetStop().GetTokenIndex()
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -135,7 +140,10 @@ func splitByParser(statement string) ([]base.Statement, error) {
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: false,
})
byteOffset += stmtByteLength
@@ -153,7 +161,8 @@ func splitByParser(statement string) ([]base.Statement, error) {
pos := goStmt.GetStop().GetTokenIndex()
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -166,7 +175,10 @@ func splitByParser(statement string) ([]base.Statement, error) {
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: false,
})
byteOffset += stmtByteLength
@@ -186,7 +198,8 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
pos := sqlClause.GetStop().GetTokenIndex()
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -199,7 +212,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: false,
})
byteOffset += stmtByteLength
@@ -209,7 +225,8 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
pos := b.Batch_level_statement().GetStop().GetTokenIndex()
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -222,7 +239,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: false,
})
byteOffset += stmtByteLength
@@ -231,7 +251,8 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
pos := b.Execute_body_batch().GetStop().GetTokenIndex()
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -244,7 +265,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: false,
})
byteOffset += stmtByteLength
@@ -253,7 +277,8 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
pos := sqlClause.GetStop().GetTokenIndex()
stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
stmtByteLength := len(stmtText)
- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
+ // Calculate start position from byte offset (first character of Text)
+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)
result = append(result, base.Statement{
Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
@@ -266,7 +291,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
int32(tokens[pos].GetColumn()),
tokens[pos].GetText(),
),
- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
+ Start: &storepb.Position{
+ Line: int32(startLine + 1),
+ Column: int32(startColumn + 1),
+ },
Empty: false,
})
byteOffset += stmtByteLength
diff --git a/backend/plugin/parser/tsql/test-data/test_split.yaml b/backend/plugin/parser/tsql/test-data/test_split.yaml
index b41757c1b5..8da8a5560e 100644
--- a/backend/plugin/parser/tsql/test-data/test_split.yaml
+++ b/backend/plugin/parser/tsql/test-data/test_split.yaml
@@ -1,71 +1,186 @@
-- description: "GO batch separator with multiple batches"
+- description: GO batch separator with multiple batches
input: "-- first statement\ndeclare @temp table(a int)\n-- second statement\ninsert into @temp values(1)\n-- third statement\nselect * from @temp\n-- go statement\ngo\ngo\ngo\ngo\n\t\t\t"
result:
- text: "-- first statement\ndeclare @temp table(a int)"
baseline: 0
- start: {line: 2, column: 1}
- end: {line: 2, column: 27}
- range: {start: 0, end: 45}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 27
+ range:
+ start: 0
+ end: 45
empty: false
- text: "\n-- second statement\ninsert into @temp values(1)"
baseline: 1
- start: {line: 4, column: 1}
- end: {line: 4, column: 28}
- range: {start: 45, end: 93}
+ start:
+ line: 2
+ column: 27
+ end:
+ line: 4
+ column: 28
+ range:
+ start: 45
+ end: 93
empty: false
- text: "\n-- third statement\nselect * from @temp\n-- go statement\ngo"
baseline: 3
- start: {line: 6, column: 1}
- end: {line: 8, column: 3}
- range: {start: 93, end: 151}
+ start:
+ line: 4
+ column: 28
+ end:
+ line: 8
+ column: 3
+ range:
+ start: 93
+ end: 151
empty: false
- text: "\ngo"
baseline: 7
- start: {line: 9, column: 1}
- end: {line: 9, column: 3}
- range: {start: 151, end: 154}
+ start:
+ line: 8
+ column: 3
+ end:
+ line: 9
+ column: 3
+ range:
+ start: 151
+ end: 154
empty: false
- text: "\ngo\ngo"
baseline: 8
- start: {line: 10, column: 1}
- end: {line: 11, column: 3}
- range: {start: 154, end: 160}
+ start:
+ line: 9
+ column: 3
+ end:
+ line: 11
+ column: 3
+ range:
+ start: 154
+ end: 160
empty: false
-
-- description: "multiple UPDATE statements with blank lines"
- input: "\n\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";\n\nUPDATE SalesLT.Address SET PostalCode = 0;\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";\n\n\n\n"
+- description: multiple UPDATE statements with blank lines
+ input: "\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";\n\nUPDATE SalesLT.Address SET PostalCode = 0;\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";\n\n"
result:
- - text: "\n\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";"
+ - text: "\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";"
baseline: 0
- start: {line: 10, column: 1}
- end: {line: 10, column: 46}
- range: {start: 0, end: 54}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 9
+ column: 46
+ range:
+ start: 0
+ end: 53
empty: false
- text: "\n\nUPDATE SalesLT.Address SET PostalCode = 0;"
- baseline: 9
- start: {line: 12, column: 1}
- end: {line: 12, column: 43}
- range: {start: 54, end: 98}
+ baseline: 8
+ start:
+ line: 9
+ column: 46
+ end:
+ line: 11
+ column: 43
+ range:
+ start: 53
+ end: 97
empty: false
- text: "\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";"
- baseline: 11
- start: {line: 15, column: 1}
- end: {line: 15, column: 69}
- range: {start: 98, end: 169}
+ baseline: 10
+ start:
+ line: 11
+ column: 43
+ end:
+ line: 14
+ column: 69
+ range:
+ start: 97
+ end: 168
empty: false
-
-- description: "unicode in table names"
+- description: unicode in table names
input: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);"
result:
- text: "SELECT * FROM 表名;"
baseline: 0
- start: {line: 1, column: 1}
- end: {line: 1, column: 18}
- range: {start: 0, end: 21}
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 18
+ range:
+ start: 0
+ end: 21
empty: false
- text: " INSERT INTO 表 VALUES (1);"
baseline: 0
- start: {line: 1, column: 19}
- end: {line: 1, column: 44}
- range: {start: 21, end: 49}
+ start:
+ line: 1
+ column: 18
+ end:
+ line: 1
+ column: 44
+ range:
+ start: 21
+ end: 49
+ empty: false
+- description: "position semantic: leading newlines"
+ input: "\n\nSELECT 1;"
+ result:
+ - text: "\n\nSELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 10
+ range:
+ start: 0
+ end: 11
+ empty: false
+- description: "position semantic: leading spaces and newlines"
+ input: "\n \n SELECT 1;"
+ result:
+ - text: "\n \n SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 0
+ end: 15
+ empty: false
+- description: "position semantic: multi-statement with leading whitespace"
+ input: "SELECT 1;\n\n SELECT 2;"
+ result:
+ - text: "SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\n\n SELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 3
+ column: 12
+ range:
+ start: 9
+ end: 22
empty: false
diff --git a/backend/plugin/schema/oracle/generate_migration_testcontainer_test.go b/backend/plugin/schema/oracle/generate_migration_testcontainer_test.go
index 6fe450da91..22bd58ec99 100644
--- a/backend/plugin/schema/oracle/generate_migration_testcontainer_test.go
+++ b/backend/plugin/schema/oracle/generate_migration_testcontainer_test.go
@@ -1874,8 +1874,18 @@ func executeStatements(db *sql.DB, statements string) error {
continue
}
- // Skip pure comment lines
- if strings.HasPrefix(stmt, "--") {
+ // Skip statements that contain only comments
+ // Strip all comment lines and check if there's actual SQL
+ lines := strings.Split(stmt, "\n")
+ hasSQL := false
+ for _, line := range lines {
+ trimmed := strings.TrimSpace(line)
+ if trimmed != "" && !strings.HasPrefix(trimmed, "--") {
+ hasSQL = true
+ break
+ }
+ }
+ if !hasSQL {
continue
}
diff --git a/backend/plugin/schema/tidb/walk_through.go b/backend/plugin/schema/tidb/walk_through.go
index 21db90525b..5ed56f8034 100644
--- a/backend/plugin/schema/tidb/walk_through.go
+++ b/backend/plugin/schema/tidb/walk_through.go
@@ -946,7 +946,9 @@ func tidbCreateTable(d *model.DatabaseMetadata, node *tidbast.CreateTableStmt) *
}
if node.Select != nil {
- content := fmt.Sprintf("CREATE TABLE AS statement is used in \"%s\"", node.Text())
+ // Trim leading whitespace for display - node.Text() may include leading whitespace
+ // to maintain position consistency, but error messages should show clean SQL.
+ content := fmt.Sprintf("CREATE TABLE AS statement is used in \"%s\"", strings.TrimSpace(node.Text()))
return &storepb.Advice{
Status: storepb.Advice_WARNING,
Code: code.StatementCreateTableAs.Int32(),