Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(parser): make Start position point to first character of Text (#18694)
h3n4l1725f34f91da
diff --git a/backend/plugin/advisor/tidb/test/column_current_time_count_limit.yaml b/backend/plugin/advisor/tidb/test/column_current_time_count_limit.yamlindex fdfb13f2c5..438d2c561a 100644--- a/backend/plugin/advisor/tidb/test/column_current_time_count_limit.yaml+++ b/backend/plugin/advisor/tidb/test/column_current_time_count_limit.yaml@@ -13,7 +13,7 @@title: COLUMN_CURRENT_TIME_COUNT_LIMITcontent: Table `t` has 3 DEFAULT CURRENT_TIMESTAMP() columns. The count greater than 2.startposition:- line: 5+ line: 1column: 0endposition: null- statement: |-@@ -28,7 +28,7 @@title: COLUMN_CURRENT_TIME_COUNT_LIMITcontent: Table `t` has 2 ON UPDATE CURRENT_TIMESTAMP() columns. The count greater than 1.startposition:- line: 4+ line: 1column: 0endposition: null- statement: |-diff --git a/backend/plugin/advisor/tidb/test/column_required.yaml b/backend/plugin/advisor/tidb/test/column_required.yamlindex 6a71e65c51..69c2d99333 100644--- a/backend/plugin/advisor/tidb/test/column_required.yaml+++ b/backend/plugin/advisor/tidb/test/column_required.yaml@@ -113,7 +113,7 @@title: COLUMN_REQUIREDcontent: 'Table `book` requires columns: updater_id'startposition:- line: 5+ line: 1column: 0endposition: null- statement: |-@@ -142,7 +142,7 @@title: COLUMN_REQUIREDcontent: 'Table `book` requires columns: creator_id'startposition:- line: 5+ line: 1column: 0endposition: null- status: 2@@ -150,7 +150,7 @@title: COLUMN_REQUIREDcontent: 'Table `student` requires columns: creator_id, updater_id'startposition:- line: 9+ line: 6column: 0endposition: null- statement: |-diff --git a/backend/plugin/advisor/tidb/test/index_total_number_limit.yaml b/backend/plugin/advisor/tidb/test/index_total_number_limit.yamlindex 80dee17c33..1006c21f6c 100644--- a/backend/plugin/advisor/tidb/test/index_total_number_limit.yaml+++ b/backend/plugin/advisor/tidb/test/index_total_number_limit.yaml@@ -52,6 +52,6 @@title: INDEX_TOTAL_NUMBER_LIMITcontent: The count of index in table `t` should be no more than 5, but found 6startposition:- line: 9+ line: 1column: 0endposition: nulldiff --git a/backend/plugin/advisor/tidb/test/table_disallow_partition.yaml b/backend/plugin/advisor/tidb/test/table_disallow_partition.yamlindex cca95a356c..bc44903ea6 100644--- a/backend/plugin/advisor/tidb/test/table_disallow_partition.yaml+++ b/backend/plugin/advisor/tidb/test/table_disallow_partition.yaml@@ -16,7 +16,7 @@PARTITION p1 VALUES LESS THAN (11))" createsstartposition:- line: 4+ line: 1column: 0endposition: null- statement: |-@@ -35,6 +35,6 @@PARTITION p1 VALUES LESS THAN (11))" createsstartposition:- line: 4+ line: 1column: 0endposition: nulldiff --git a/backend/plugin/parser/base/lexer.go b/backend/plugin/parser/base/lexer.goindex a1553b568f..fa7271d622 100644--- a/backend/plugin/parser/base/lexer.go+++ b/backend/plugin/parser/base/lexer.go@@ -60,10 +60,13 @@ func IsEmpty(tokens []antlr.Token, semi int) bool {// SplitSQLByLexer is a grammar-free helper function that splits SQL statements using an ANTLR lexer.// It works with any ANTLR-based lexer by accepting the token stream and semicolon token type as parameters.//+// IMPORTANT: This function requires that the lexer sends whitespace to the hidden channel+// (using `-> channel(HIDDEN)` in the grammar), NOT skipping it. This ensures GetAllTokens()+// returns all tokens including whitespace, allowing us to use token positions directly.+//// Parameters:// - stream: The ANTLR token stream (must be already filled with stream.Fill())// - semiTokenType: The token type value for semicolon in the specific grammar-// - statement: The original SQL statement string (used for position conversion)//// Returns:// - A slice of Statement, each representing one statement with its text, position, and metadata@@ -73,11 +76,12 @@ func IsEmpty(tokens []antlr.Token, semi int) bool {// lexer := parser.NewSnowflakeLexer(antlr.NewInputStream(statement))// stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)// stream.Fill()-// return SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI, statement)-func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, statement string) ([]Statement, error) {+// return SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI)+func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int) ([]Statement, error) {tokens := stream.GetAllTokens()var buf []antlr.Tokenvar sqls []Statement+ byteOffset := 0for i, token := range tokens {// Collect all tokens except the last one (EOF)@@ -101,27 +105,32 @@ func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, stateme}}- // Get the position of the first default channel token for Start position- antlrPosition := FirstDefaultChannelTokenPosition(buf)+ stmtText := bufStr.String()+ stmtByteLength := len(stmtText)// Use the last token in the buffer for End position (not EOF token when at end of stream)lastToken := buf[len(buf)-1]+ // Use buf[0]'s position directly since GetAllTokens() includes hidden channel tokens (whitespace/comments)sqls = append(sqls, Statement{- Text: bufStr.String(),+ Text: stmtText,BaseLine: buf[0].GetLine() - 1, // BaseLine is the offset of the first tokenRange: &storepb.Range{- Start: int32(buf[0].GetStart()),- End: int32(lastToken.GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRTokenToExclusiveEndPosition(int32(lastToken.GetLine()),int32(lastToken.GetColumn()),lastToken.GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(buf[0].GetLine()),+ Column: int32(buf[0].GetColumn() + 1),+ },Empty: empty,})+ byteOffset += stmtByteLengthbuf = nilcontinue}@@ -129,3 +138,21 @@ func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, statemereturn sqls, nil}++// CalculateLineAndColumn calculates the 0-based line number and 0-based column (character offset)+// for a given byte offset in the statement.+func CalculateLineAndColumn(statement string, byteOffset int) (line, column int) {+ if byteOffset > len(statement) {+ byteOffset = len(statement)+ }+ // Range over string iterates over runes (code points), not bytes+ for _, r := range statement[:byteOffset] {+ if r == '\n' {+ line+++ column = 0+ } else {+ column+++ }+ }+ return line, column+}diff --git a/backend/plugin/parser/bigquery/split.go b/backend/plugin/parser/bigquery/split.goindex 0a86d05160..f296bc7512 100644--- a/backend/plugin/parser/bigquery/split.go+++ b/backend/plugin/parser/bigquery/split.go@@ -1,12 +1,9 @@package bigqueryimport (- "strings"-"github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/googlesql"- "github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base")@@ -19,49 +16,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {lexer := parser.NewGoogleSQLLexer(antlr.NewInputStream(statement))stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()-- tokens := stream.GetAllTokens()- var buf []antlr.Token- var sqls []base.Statement- byteOffset := 0- for i, token := range tokens {- if i < len(tokens)-1 {- buf = append(buf, token)- }- if (token.GetTokenType() == parser.GoogleSQLLexerSEMI_SYMBOL || i == len(tokens)-1) && len(buf) > 0 {- bufStr := new(strings.Builder)- empty := true-- for _, b := range buf {- if _, err := bufStr.WriteString(b.GetText()); err != nil {- return nil, err- }- if b.GetChannel() != antlr.TokenHiddenChannel {- empty = false- }- }- stmtText := bufStr.String()- stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(buf)- sqls = append(sqls, base.Statement{- Text: stmtText,- BaseLine: buf[0].GetLine() - 1,- Range: &storepb.Range{- Start: int32(byteOffset),- End: int32(byteOffset + stmtByteLength),- },- End: common.ConvertANTLRTokenToExclusiveEndPosition(- int32(buf[len(buf)-1].GetLine()),- int32(buf[len(buf)-1].GetColumn()),- buf[len(buf)-1].GetText(),- ),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),- Empty: empty,- })- byteOffset += stmtByteLength- buf = nil- continue- }- }- return sqls, nil+ return base.SplitSQLByLexer(stream, parser.GoogleSQLLexerSEMI_SYMBOL)}diff --git a/backend/plugin/parser/bigquery/test-data/test_split.yaml b/backend/plugin/parser/bigquery/test-data/test_split.yamlindex 430a24714b..57c00db271 100644--- a/backend/plugin/parser/bigquery/test-data/test_split.yaml+++ b/backend/plugin/parser/bigquery/test-data/test_split.yaml@@ -1,25 +1,99 @@-- description: "single statement without semicolon"- input: "SELECT 1"+- description: single statement without semicolon+ input: SELECT 1result:- - text: "SELECT 1"+ - text: SELECT 1baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 9}- range: {start: 0, end: 8}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 9+ range:+ start: 0+ end: 8empty: false--- description: "multiple statements multi-line"+- description: multiple statements multi-lineinput: "SELECT 1;\n SELECT\n 33;"result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- text: "\n SELECT\n 33;"baseline: 0- start: {line: 2, column: 2}- end: {line: 3, column: 5}- range: {start: 9, end: 22}+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 5+ range:+ start: 9+ end: 22+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/cassandra/split.go b/backend/plugin/parser/cassandra/split.goindex 3146b8ceff..411cbc257c 100644--- a/backend/plugin/parser/cassandra/split.go+++ b/backend/plugin/parser/cassandra/split.go@@ -18,5 +18,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()- return base.SplitSQLByLexer(stream, cql.CqlLexerSEMI, statement)+ return base.SplitSQLByLexer(stream, cql.CqlLexerSEMI)}diff --git a/backend/plugin/parser/cassandra/test-data/test_split.yaml b/backend/plugin/parser/cassandra/test-data/test_split.yamlindex 0043d41600..60ded7187e 100644--- a/backend/plugin/parser/cassandra/test-data/test_split.yaml+++ b/backend/plugin/parser/cassandra/test-data/test_split.yaml@@ -1,31 +1,111 @@-- description: "multiple statements on multiple lines"+- description: multiple statements on multiple linesinput: "SELECT * FROM users;\n\tSELECT * FROM orders;\n\tSELECT * FROM products;"result:- - text: "SELECT * FROM users;"+ - text: SELECT * FROM users;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 21}- range: {start: 0, end: 20}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 21+ range:+ start: 0+ end: 20empty: false- text: "\n\tSELECT * FROM orders;"baseline: 0- start: {line: 2, column: 2}- end: {line: 2, column: 23}- range: {start: 20, end: 43}+ start:+ line: 1+ column: 21+ end:+ line: 2+ column: 23+ range:+ start: 20+ end: 43empty: false- text: "\n\tSELECT * FROM products;"baseline: 1- start: {line: 3, column: 2}- end: {line: 3, column: 25}- range: {start: 43, end: 68}+ start:+ line: 2+ column: 23+ end:+ line: 3+ column: 25+ range:+ start: 43+ end: 68empty: false--- description: "single statement"- input: "SELECT * FROM users;"+- description: single statement+ input: SELECT * FROM users;result:- - text: "SELECT * FROM users;"+ - text: SELECT * FROM users;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 21}- range: {start: 0, end: 20}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 21+ range:+ start: 0+ end: 20+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/cosmosdb/split.go b/backend/plugin/parser/cosmosdb/split.goindex 2eb1f1f6ae..6afe56f0f8 100644--- a/backend/plugin/parser/cosmosdb/split.go+++ b/backend/plugin/parser/cosmosdb/split.go@@ -46,26 +46,22 @@ func SplitSQL(statement string) ([]base.Statement, error) {}}- var firstToken, lastToken antlr.Token+ var lastToken antlr.Tokenfor _, token := range tokens {if token.GetTokenType() == antlr.TokenEOF {break}- if firstToken == nil && token.GetChannel() == antlr.TokenDefaultChannel {- firstToken = token- }if token.GetChannel() == antlr.TokenDefaultChannel {lastToken = token}}- if firstToken == nil {- firstToken = tokens[0]- }if lastToken == nil && len(tokens) > 0 {lastToken = tokens[len(tokens)-1]}+ // Since Text is the entire statement (including any leading whitespace),+ // Start should point to the first character (position 1,1)return []base.Statement{{Text: statement,@@ -74,10 +70,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {Start: 0,End: int32(len(statement)),},- Start: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{- Line: int32(firstToken.GetLine()),- Column: int32(firstToken.GetColumn()),- }, statement),+ Start: &storepb.Position{+ Line: 1,+ Column: 1,+ },End: common.ConvertANTLRTokenToExclusiveEndPosition(int32(lastToken.GetLine()),int32(lastToken.GetColumn()),diff --git a/backend/plugin/parser/cosmosdb/test-data/test_split.yaml b/backend/plugin/parser/cosmosdb/test-data/test_split.yamlindex 8524b067ad..50287e2a78 100644--- a/backend/plugin/parser/cosmosdb/test-data/test_split.yaml+++ b/backend/plugin/parser/cosmosdb/test-data/test_split.yaml@@ -1,29 +1,79 @@-- description: "single SELECT statement"- input: "SELECT * FROM c"+- description: single SELECT statement+ input: SELECT * FROM cresult:- - text: "SELECT * FROM c"+ - text: SELECT * FROM cbaseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 16}- range: {start: 0, end: 15}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 16+ range:+ start: 0+ end: 15empty: false--- description: "SELECT with WHERE"- input: "SELECT c.id FROM users c WHERE c.active = true"+- description: SELECT with WHERE+ input: SELECT c.id FROM users c WHERE c.active = trueresult:- - text: "SELECT c.id FROM users c WHERE c.active = true"+ - text: SELECT c.id FROM users c WHERE c.active = truebaseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 47}- range: {start: 0, end: 46}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 47+ range:+ start: 0+ end: 46empty: false--- description: "multi-line SELECT"- input: "SELECT\n * FROM c"+- description: multi-line SELECT+ input: |-+ SELECT+ * FROM cresult:- - text: "SELECT\n * FROM c"+ - text: |-+ SELECT+ * FROM cbaseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 11}- range: {start: 0, end: 17}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 11+ range:+ start: 0+ end: 17+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1"+ result:+ - text: "\n\nSELECT 1"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 9+ range:+ start: 0+ end: 10+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1"+ result:+ - text: "\n \n SELECT 1"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 11+ range:+ start: 0+ end: 14empty: falsediff --git a/backend/plugin/parser/doris/split.go b/backend/plugin/parser/doris/split.goindex 39cd3ef308..7ee49fc059 100644--- a/backend/plugin/parser/doris/split.go+++ b/backend/plugin/parser/doris/split.go@@ -19,5 +19,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()- return base.SplitSQLByLexer(stream, parser.DorisLexerSEMICOLON, statement)+ return base.SplitSQLByLexer(stream, parser.DorisLexerSEMICOLON)}diff --git a/backend/plugin/parser/doris/test-data/test_split.yaml b/backend/plugin/parser/doris/test-data/test_split.yamlindex 3d29e14d8c..7459717ad9 100644--- a/backend/plugin/parser/doris/test-data/test_split.yaml+++ b/backend/plugin/parser/doris/test-data/test_split.yaml@@ -1,51 +1,141 @@-- description: "simple single statement"- input: "SELECT 1;"+- description: simple single statement+ input: SELECT 1;result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false--- description: "multiple statements on one line"- input: "SELECT 1; SELECT 2;"+- description: multiple statements on one line+ input: SELECT 1; SELECT 2;result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- - text: " SELECT 2;"+ - text: ' SELECT 2;'baseline: 0- start: {line: 1, column: 11}- end: {line: 1, column: 20}- range: {start: 9, end: 19}+ start:+ line: 1+ column: 10+ end:+ line: 1+ column: 20+ range:+ start: 9+ end: 19empty: false--- description: "multiple statements on multiple lines"+- description: multiple statements on multiple linesinput: "SELECT 1;\nSELECT 2;"result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- text: "\nSELECT 2;"baseline: 0- start: {line: 2, column: 1}- end: {line: 2, column: 10}- range: {start: 9, end: 19}+ start:+ line: 1+ column: 10+ end:+ line: 2+ column: 10+ range:+ start: 9+ end: 19empty: false--- description: "multi-line statement"+- description: multi-line statementinput: "SELECT\n 1;"result:- text: "SELECT\n 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 5}- range: {start: 0, end: 11}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 5+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/mysql/mysql.go b/backend/plugin/parser/mysql/mysql.goindex e1f534d6a4..6c182ae53b 100644--- a/backend/plugin/parser/mysql/mysql.go+++ b/backend/plugin/parser/mysql/mysql.go@@ -1,7 +1,6 @@package mysqlimport (- "fmt""log/slog""regexp""strings"@@ -95,7 +94,7 @@ func DealWithDelimiter(statement string) (string, error) {return "", err}if has {- var result []string+ var result strings.Builderdelimiter := `;`for _, sql := range list {if IsDelimiter(sql.Text) {@@ -103,99 +102,30 @@ func DealWithDelimiter(statement string) (string, error) {if err != nil {return "", err}- result = append(result, "-- "+sql.Text)+ // Comment out only the DELIMITER line, preserving all other lines for correct line numbers+ lines := strings.Split(sql.Text, "\n")+ for i, line := range lines {+ if IsDelimiter(line) {+ lines[i] = "-- " + strings.TrimLeft(line, " \t")+ break+ }+ }+ result.WriteString(strings.Join(lines, "\n"))continue}- // TODO(rebelice): after deal with delimiter, we may cannot get the right line number, fix it.if delimiter != ";" && !sql.Empty {- result = append(result, fmt.Sprintf("%s;", strings.TrimSuffix(sql.Text, delimiter)))+ result.WriteString(strings.TrimSuffix(sql.Text, delimiter))+ result.WriteString(";")} else {- result = append(result, sql.Text)+ result.WriteString(sql.Text)}}- statement = strings.Join(result, "\n")+ statement = result.String()}return statement, nil}-// RestoreDelimiter tries to restore the delimiter statement which is commented by DealWithDelimiter,-//-// Assumes the delimiter comment takes up a single line and matches the syntax `-- DELIMITER ?`,-// also, one delimiter comment MUST only affect one statement, in the other words, the following statement cause UB:-// -- DELIMITER ;-// SELECT 1;-// SELECT 2;-// -- DELIMITER ;.-// If the current delimiter is `;`, and meet `-- DELIMITER ;`, we would skip this delimiter comment.-func RestoreDelimiter(statement string) (string, error) {- delimiterRegexp := regexp.MustCompile(`(?i)^-- DELIMITER\s+(?P<DELIMITER>[^\s\\]+)\s*`)- lines := strings.Split(statement, "\n")- var result strings.Builder- var buf strings.Builder- previousDelimiter := ";"-- flushBuf := func(delimiter string) error {- bufStmt := buf.String()- buf.Reset()- // Lookup reversely to modify the last delimiter.- for i := len(bufStmt) - 1; i >= 0; i-- {- if bufStmt[i] == ';' {- bufStmt = bufStmt[:i] + delimiter + bufStmt[i+1:]- break- }- }- if _, err := result.WriteString(bufStmt); err != nil {- return errors.Wrapf(err, "failed to write string %q into builder", bufStmt)- }- return nil- }-- for i, line := range lines {- matchList := delimiterRegexp.FindStringSubmatch(line)- if matchList == nil {- target := &result- if previousDelimiter != ";" {- target = &buf- }- // Like strings.Join.- s := line- if i != len(lines)-1 {- s = fmt.Sprintf("%s\n", line)- }- if _, err := target.WriteString(s); err != nil {- return "", errors.Wrapf(err, "failed to write string %q into builder", line)- }- continue- }- if len(matchList) != 2 {- return "", errors.Errorf("invalid delimiter comment %q", line)- }-- // If meet delimiter again, we should convert the delimiter in buf to previous delimiter.- if previousDelimiter != ";" {- if err := flushBuf(previousDelimiter); err != nil {- return "", err- }- }-- if matchList[1] != previousDelimiter {- if _, err := result.WriteString(fmt.Sprintf("DELIMITER %s\n", matchList[1])); err != nil {- return "", errors.Wrapf(err, "failed to write string %q into builder", line)- }- }- previousDelimiter = matchList[1]- }-- if previousDelimiter != ";" {- if err := flushBuf(previousDelimiter); err != nil {- return "", err- }- }-- return result.String(), nil-}-func parseSingleStatement(baseLine int, statement string) (antlr.Tree, *antlr.CommonTokenStream, error) {input := antlr.NewInputStream(statement)lexer := parser.NewMySQLLexer(input)diff --git a/backend/plugin/parser/mysql/mysql_test.go b/backend/plugin/parser/mysql/mysql_test.goindex 96f7a129b4..ba7231e0ba 100644--- a/backend/plugin/parser/mysql/mysql_test.go+++ b/backend/plugin/parser/mysql/mysql_test.go@@ -155,300 +155,6 @@ func TestMySQLParser(t *testing.T) {}}-// func TestDealWithDelimiter_SplitCommentBeforeDelimiter(t *testing.T) {-// tests := []struct {-// input string-// want string-// }{-// {-// input: `-// DELIMITER ;;-// CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4-// DETERMINISTIC-// RETURN CONCAT('Hello, ',s,'!') ;;-// DELIMITER ;-// ---// -- Function structure for hello2-// ---// DELIMITER ;;-// CREATE FUNCTION hello2(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4-// DETERMINISTIC-// RETURN CONCAT('Hello, ',s,'!') ;;-// DELIMITER ;-// `,-// want: `-- DELIMITER ;;-// CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4-// DETERMINISTIC-// RETURN CONCAT('Hello, ',s,'!') ;-// -- DELIMITER ;-// ---// -- Function structure for hello2-// ---// -- DELIMITER ;;-// CREATE FUNCTION hello2(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4-// DETERMINISTIC-// RETURN CONCAT('Hello, ',s,'!') ;-// -- DELIMITER ;`,-// },-// }--// a := require.New(t)-// for i, test := range tests {-// got, err := DealWithDelimiter(test.input, tokenizer.SplitCommentBeforeDelimiter())-// a.NoError(err, i)-// a.Equal(test.want, got, i)-// }-// }--func TestRestoreDelimiter(t *testing.T) {- testCases := []struct {- input string- want string- }{- {- // No delimiter- input: `-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;------ Table structure for u----CREATE TABLE u (- b blob NOT NULL,- t text NOT NULL,- g geometry NOT NULL,- j json NOT NULL,- ti tinyint(1) NOT NULL,- tb tinyblob-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;-`,- want: `-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;------ Table structure for u----CREATE TABLE u (- b blob NOT NULL,- t text NOT NULL,- g geometry NOT NULL,- j json NOT NULL,- ti tinyint(1) NOT NULL,- tb tinyblob-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;-`,- },-- {- // Semicolon in body after delimiter- input: `-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;------ Table structure for u----CREATE TABLE u (- b blob NOT NULL,- t text NOT NULL,- g geometry NOT NULL,- j json NOT NULL,- ti tinyint(1) NOT NULL,- tb tinyblob-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;---- DELIMITER ;;-CREATE DEFINER=root@localhost PROCEDURE GetEmployeeFullName(IN in_employee_id INT, OUT out_full_name VARCHAR(100))-BEGIN- -- Declare variables- DECLARE first_name VARCHAR(50);- DECLARE last_name VARCHAR(50);-- -- Initialize variables- SET out_full_name = NULL;-- -- Retrieve employee details based on employee_id- SELECT first_name, last_name INTO first_name, last_name- FROM employees- WHERE employee_id = in_employee_id;-- -- Check if employee exists- IF first_name IS NOT NULL THEN- -- Concatenate first and last names- SET out_full_name = CONCAT(first_name, ' ', last_name);- END IF;-END ;--- DELIMITER ;-`,- want: `-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;------ Table structure for u----CREATE TABLE u (- b blob NOT NULL,- t text NOT NULL,- g geometry NOT NULL,- j json NOT NULL,- ti tinyint(1) NOT NULL,- tb tinyblob-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;--DELIMITER ;;-CREATE DEFINER=root@localhost PROCEDURE GetEmployeeFullName(IN in_employee_id INT, OUT out_full_name VARCHAR(100))-BEGIN- -- Declare variables- DECLARE first_name VARCHAR(50);- DECLARE last_name VARCHAR(50);-- -- Initialize variables- SET out_full_name = NULL;-- -- Retrieve employee details based on employee_id- SELECT first_name, last_name INTO first_name, last_name- FROM employees- WHERE employee_id = in_employee_id;-- -- Check if employee exists- IF first_name IS NOT NULL THEN- -- Concatenate first and last names- SET out_full_name = CONCAT(first_name, ' ', last_name);- END IF;-END ;;-DELIMITER ;-`,- },- {- // One delimiter- input: `-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;------ Table structure for u----CREATE TABLE u (- b blob NOT NULL,- t text NOT NULL,- g geometry NOT NULL,- j json NOT NULL,- ti tinyint(1) NOT NULL,- tb tinyblob-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;--- DELIMITER ;;-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;-`,- want: `-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;------ Table structure for u----CREATE TABLE u (- b blob NOT NULL,- t text NOT NULL,- g geometry NOT NULL,- j json NOT NULL,- ti tinyint(1) NOT NULL,- tb tinyblob-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;-DELIMITER ;;-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;;-`,- },-- {- // Multiple delimiters.- input: `-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;------ Table structure for u----CREATE TABLE u (- b blob NOT NULL,- t text NOT NULL,- g geometry NOT NULL,- j json NOT NULL,- ti tinyint(1) NOT NULL,- tb tinyblob-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;--- DELIMITER ;;-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;--- DELIMITER ;-CREATE FUNCTION hello2(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;--- DELIMITER ??-CREATE FUNCTION hello3(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;--- DELIMITER ;-DROP FUNCTION hello3;-`,- want: `-SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;-SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;------ Table structure for u----CREATE TABLE u (- b blob NOT NULL,- t text NOT NULL,- g geometry NOT NULL,- j json NOT NULL,- ti tinyint(1) NOT NULL,- tb tinyblob-) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_0900_ai_ci;-DELIMITER ;;-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;;-DELIMITER ;-CREATE FUNCTION hello2(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;-DELIMITER ??-CREATE FUNCTION hello3(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ??-DELIMITER ;-DROP FUNCTION hello3;-`,- },-- // Ignore redundant DELIMITER statements.- {- input: `--- DELIMITER ;;-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;--- DELIMITER ;--- DELIMITER ;-`,- want: `-DELIMITER ;;-CREATE FUNCTION hello(s CHAR(20)) RETURNS char(50) CHARSET utf8mb4- DETERMINISTIC-RETURN CONCAT('Hello, ',s,'!') ;;-DELIMITER ;-`,- },- }-- for i, tc := range testCases {- got, err := RestoreDelimiter(tc.input)- require.NoError(t, err)- require.Equalf(t, tc.want, got, "test cases: %d", i)- }-}-func TestParseMySQLStatements(t *testing.T) {statement := "SELECT 1; SELECT 2;"diff --git a/backend/plugin/parser/mysql/split.go b/backend/plugin/parser/mysql/split.goindex 84a5e1885d..7d943a51f7 100644--- a/backend/plugin/parser/mysql/split.go+++ b/backend/plugin/parser/mysql/split.go@@ -66,7 +66,8 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([stmtStartByte := len(statement[:stmtStartChar])stmtEndByte := len(statement[:stmtEndChar])- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : i+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, stmtStartByte)// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{@@ -81,7 +82,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([int32(tokens[i].GetColumn()),tokens[i].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:i+1], parser.MySQLLexerSEMICOLON_SYMBOL),})i++@@ -103,7 +107,8 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([stmtStartByte := len(statement[:stmtStartChar])stmtEndByte := len(statement[:stmtEndChar])- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:newStart])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, stmtStartByte)// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{@@ -119,7 +124,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([int32(tokens[newStart-1].GetColumn()),tokens[newStart-1].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:i], parser.MySQLLexerSEMICOLON_SYMBOL),})i = newStart@@ -140,7 +148,8 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([stmtStartByte := len(statement[:stmtStartChar])stmtEndByte := len(statement[:stmtEndChar])- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, stmtStartByte)// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{@@ -155,7 +164,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([int32(tokens[endPos-1].GetColumn()),tokens[endPos-1].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:endPos], parser.MySQLLexerSEMICOLON_SYMBOL),})}@@ -241,7 +253,8 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.ComstmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{@@ -256,7 +269,10 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Comint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),})byteOffset += stmtByteLength@@ -268,7 +284,8 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.ComstmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{@@ -283,7 +300,10 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Comint32(tokens[eofPos-1].GetColumn()),tokens[eofPos-1].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:eofPos], parser.MySQLLexerSEMICOLON_SYMBOL),})}@@ -437,7 +457,8 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]bstmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{@@ -452,7 +473,10 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]bint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),})byteOffset += stmtByteLength@@ -464,7 +488,8 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]bstmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{@@ -479,7 +504,10 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]bint32(tokens[eofPos-1].GetColumn()),tokens[eofPos-1].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:eofPos], parser.MySQLLexerSEMICOLON_SYMBOL),})}diff --git a/backend/plugin/parser/mysql/test-data/test_backup.yaml b/backend/plugin/parser/mysql/test-data/test_backup.yamlindex 08ba863637..d6f6ae0d69 100644--- a/backend/plugin/parser/mysql/test-data/test_backup.yaml+++ b/backend/plugin/parser/mysql/test-data/test_backup.yaml@@ -160,8 +160,8 @@sourcetablename: t2targettablename: _rollback_t2_db2startposition:- line: 2- column: 1+ line: 1+ column: 39endposition:line: 2column: 39diff --git a/backend/plugin/parser/mysql/test-data/test_split.yaml b/backend/plugin/parser/mysql/test-data/test_split.yamlindex 2c2b35db29..84c8204ea0 100644--- a/backend/plugin/parser/mysql/test-data/test_split.yaml+++ b/backend/plugin/parser/mysql/test-data/test_split.yaml@@ -1,243 +1,480 @@-- description: "DELIMITER with stored procedure"- input: "\n\t\t\tDELIMITER ;;\n\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\tBEGIN\n\t\t\t\tDECLARE x INT;\n\t\t\t\tSET x = 0;\n\t\t\t\tlabel1: WHILE x < p1 DO\n\t\t\t\t\tSET x = x + 1;\n\t\t\t\tEND WHILE label1;\n\t\t\tEND;;\n\t\t\tDELIMITER ;\n\t\t\tCALL dorepeat(1000);\n\t\t\tSELECT x;\n\t\t\t"+- description: DELIMITER with stored procedure+ input: "\t\t\tDELIMITER ;;\n\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\tBEGIN\n\t\t\t\tDECLARE x INT;\n\t\t\t\tSET x = 0;\n\t\t\t\tlabel1: WHILE x < p1 DO\n\t\t\t\t\tSET x = x + 1;\n\t\t\t\tEND WHILE label1;\n\t\t\tEND;;\n\t\t\tDELIMITER ;\n\t\t\tCALL dorepeat(1000);\n\t\t\tSELECT x;\n\t\t\t"result:- text: "\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\tBEGIN\n\t\t\t\tDECLARE x INT;\n\t\t\t\tSET x = 0;\n\t\t\t\tlabel1: WHILE x < p1 DO\n\t\t\t\t\tSET x = x + 1;\n\t\t\t\tEND WHILE label1;\n\t\t\tEND;"- baseline: 2- start: {line: 3, column: 4}- end: {line: 10, column: 9}- range: {start: 17, end: 175}+ baseline: 1+ start:+ line: 2+ column: 1+ end:+ line: 9+ column: 9+ range:+ start: 16+ end: 174empty: false- text: "\t\t\tCALL dorepeat(1000);"- baseline: 11- start: {line: 12, column: 4}- end: {line: 12, column: 24}- range: {start: 191, end: 214}+ baseline: 10+ start:+ line: 11+ column: 1+ end:+ line: 11+ column: 24+ range:+ start: 190+ end: 213empty: false- text: "\n\t\t\tSELECT x;"- baseline: 11- start: {line: 13, column: 4}- end: {line: 13, column: 13}- range: {start: 214, end: 227}+ baseline: 10+ start:+ line: 11+ column: 24+ end:+ line: 12+ column: 13+ range:+ start: 213+ end: 226empty: false- text: "\n\t\t\t"- baseline: 12- start: {line: 14, column: 4}- end: {line: 14, column: 4}- range: {start: 227, end: 231}+ baseline: 11+ start:+ line: 12+ column: 13+ end:+ line: 13+ column: 4+ range:+ start: 226+ end: 230empty: true--- description: "escaped double quote in string"- input: "select * from t;select \"\\\"\" where true;"+- description: escaped double quote in string+ input: select * from t;select "\"" where true;result:- - text: "select * from t;"+ - text: select * from t;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 17}- range: {start: 0, end: 16}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 17+ range:+ start: 0+ end: 16empty: false- - text: "select \"\\\"\" where true;"+ - text: select "\"" where true;baseline: 0- start: {line: 1, column: 17}- end: {line: 1, column: 40}- range: {start: 16, end: 39}+ start:+ line: 1+ column: 17+ end:+ line: 1+ column: 40+ range:+ start: 16+ end: 39empty: false--- description: "comments only creates empty statement"+- description: comments only creates empty statementinput: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n"result:- text: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n"baseline: 0- start: {line: 3, column: 1}- end: {line: 2, column: 24}- range: {start: 0, end: 39}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 24+ range:+ start: 0+ end: 39empty: true--- description: "statement followed by trailing block comment"+- description: statement followed by trailing block commentinput: "select * from t;\n\t\t\t/* sdfasdf */"result:- - text: "select * from t;"+ - text: select * from t;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 17}- range: {start: 0, end: 16}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 17+ range:+ start: 0+ end: 16empty: false- text: "\n\t\t\t/* sdfasdf */"baseline: 0- start: {line: 2, column: 17}- end: {line: 2, column: 17}- range: {start: 16, end: 33}+ start:+ line: 1+ column: 17+ end:+ line: 2+ column: 17+ range:+ start: 16+ end: 33empty: true--- description: "statement then empty comment statement then statement"+- description: statement then empty comment statement then statementinput: "select * from t;\n\t\t\t/* sdfasdf */;\n\t\t\tselect * from t;"result:- - text: "select * from t;"+ - text: select * from t;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 17}- range: {start: 0, end: 16}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 17+ range:+ start: 0+ end: 16empty: false- text: "\n\t\t\t/* sdfasdf */;"baseline: 0- start: {line: 2, column: 17}- end: {line: 2, column: 18}- range: {start: 16, end: 34}+ start:+ line: 1+ column: 17+ end:+ line: 2+ column: 18+ range:+ start: 16+ end: 34empty: true- text: "\n\t\t\tselect * from t;"baseline: 1- start: {line: 3, column: 4}- end: {line: 3, column: 20}- range: {start: 34, end: 54}+ start:+ line: 2+ column: 18+ end:+ line: 3+ column: 20+ range:+ start: 34+ end: 54+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: false--- description: "function with BEGIN END block"+- description: function with BEGIN END blockinput: "CREATE DEFINER=`root`@`%` FUNCTION `CalcIncome`( starting_value INT ) RETURNS int\nBEGIN\n\n\t\t DECLARE income INT;\n\n\t\t SET income = 0;\n\n\t\t label1: WHILE income <= 3000 DO\n\t\t\t SET income = income + starting_value;\n\t\t END WHILE label1;\n\n\t\t RETURN income;\n\n\t\tEND ;"result:- text: "CREATE DEFINER=`root`@`%` FUNCTION `CalcIncome`( starting_value INT ) RETURNS int\nBEGIN\n\n\t\t DECLARE income INT;\n\n\t\t SET income = 0;\n\n\t\t label1: WHILE income <= 3000 DO\n\t\t\t SET income = income + starting_value;\n\t\t END WHILE label1;\n\n\t\t RETURN income;\n\n\t\tEND ;"baseline: 0- start: {line: 1, column: 1}- end: {line: 14, column: 8}- range: {start: 0, end: 268}+ start:+ line: 1+ column: 1+ end:+ line: 14+ column: 8+ range:+ start: 0+ end: 268empty: false--- description: "multiple statements with leading whitespace and no trailing semicolon"- input: " CREATE TABLE t(a int); CREATE TABLE t1(a int)"+- description: multiple statements with leading whitespace and no trailing semicolon+ input: ' CREATE TABLE t(a int); CREATE TABLE t1(a int)'result:- - text: " CREATE TABLE t(a int);"+ - text: ' CREATE TABLE t(a int);'baseline: 0- start: {line: 1, column: 5}- end: {line: 1, column: 27}- range: {start: 0, end: 26}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 27+ range:+ start: 0+ end: 26empty: false- - text: " CREATE TABLE t1(a int)"+ - text: ' CREATE TABLE t1(a int)'baseline: 0- start: {line: 1, column: 28}- end: {line: 1, column: 50}- range: {start: 26, end: 49}+ start:+ line: 1+ column: 27+ end:+ line: 1+ column: 50+ range:+ start: 26+ end: 49empty: false--- description: "backtick identifiers with escaped strings"+- description: backtick identifiers with escaped stringsinput: "CREATE TABLE `tech_Book`(id int, name varchar(255));\nINSERT INTO `tech_Book` VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"result:- - text: "CREATE TABLE `tech_Book`(id int, name varchar(255));"+ - text: CREATE TABLE `tech_Book`(id int, name varchar(255));baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 53}- range: {start: 0, end: 52}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 53+ range:+ start: 0+ end: 52empty: false- text: "\nINSERT INTO `tech_Book` VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"baseline: 0- start: {line: 2, column: 1}- end: {line: 2, column: 79}- range: {start: 52, end: 131}+ start:+ line: 1+ column: 53+ end:+ line: 2+ column: 79+ range:+ start: 52+ end: 131empty: false--- description: "block comment, line comment, and hash comment"- input: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');\n\t\t\t\t\t\t# this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"+- description: block comment, line comment, and hash comment+ input: "\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');\n\t\t\t\t\t\t# this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"result:- - text: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));"+ - text: "\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));"baseline: 0- start: {line: 3, column: 7}- end: {line: 3, column: 77}- range: {start: 0, end: 110}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 77+ range:+ start: 0+ end: 109empty: false- text: "\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"- baseline: 2- start: {line: 5, column: 7}- end: {line: 5, column: 83}- range: {start: 110, end: 223}+ baseline: 1+ start:+ line: 2+ column: 77+ end:+ line: 4+ column: 83+ range:+ start: 109+ end: 222empty: false- text: "\n\t\t\t\t\t\t# this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');"- baseline: 4- start: {line: 7, column: 7}- end: {line: 7, column: 83}- range: {start: 223, end: 335}+ baseline: 3+ start:+ line: 4+ column: 83+ end:+ line: 6+ column: 83+ range:+ start: 222+ end: 334empty: false--- description: "procedure with REPEAT and hash comment"+- description: procedure with REPEAT and hash commentinput: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;\n\t\t\t\t\t\tCALL dorepeat(1000);\n\t\t\t\t\t\tSELECT @x;\n\t\t\t\t\t\t"result:- text: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;"baseline: 0- start: {line: 2, column: 7}- end: {line: 7, column: 8}- range: {start: 0, end: 181}+ start:+ line: 1+ column: 1+ end:+ line: 7+ column: 8+ range:+ start: 0+ end: 181empty: false- text: "\n\t\t\t\t\t\tCALL dorepeat(1000);"baseline: 6- start: {line: 8, column: 7}- end: {line: 8, column: 27}- range: {start: 181, end: 208}+ start:+ line: 7+ column: 8+ end:+ line: 8+ column: 27+ range:+ start: 181+ end: 208empty: false- text: "\n\t\t\t\t\t\tSELECT @x;"baseline: 7- start: {line: 9, column: 7}- end: {line: 9, column: 17}- range: {start: 208, end: 225}+ start:+ line: 8+ column: 27+ end:+ line: 9+ column: 17+ range:+ start: 208+ end: 225empty: false- text: "\n\t\t\t\t\t\t"baseline: 8- start: {line: 10, column: 7}- end: {line: 10, column: 7}- range: {start: 225, end: 232}+ start:+ line: 9+ column: 17+ end:+ line: 10+ column: 7+ range:+ start: 225+ end: 232empty: true--- description: "procedure with block comment inside"+- description: procedure with block comment insideinput: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\t/* This is a comment */\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;\n\t\t\t\t\t\tCALL dorepeat(1000);\n\t\t\t\t\t\tSELECT @x;\n\t\t\t\t\t\t"result:- text: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\t/* This is a comment */\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;"baseline: 0- start: {line: 2, column: 7}- end: {line: 8, column: 8}- range: {start: 0, end: 211}+ start:+ line: 1+ column: 1+ end:+ line: 8+ column: 8+ range:+ start: 0+ end: 211empty: false- text: "\n\t\t\t\t\t\tCALL dorepeat(1000);"baseline: 7- start: {line: 9, column: 7}- end: {line: 9, column: 27}- range: {start: 211, end: 238}+ start:+ line: 8+ column: 8+ end:+ line: 9+ column: 27+ range:+ start: 211+ end: 238empty: false- text: "\n\t\t\t\t\t\tSELECT @x;"baseline: 8- start: {line: 10, column: 7}- end: {line: 10, column: 17}- range: {start: 238, end: 255}+ start:+ line: 9+ column: 27+ end:+ line: 10+ column: 17+ range:+ start: 238+ end: 255empty: false- text: "\n\t\t\t\t\t\t"baseline: 9- start: {line: 11, column: 7}- end: {line: 11, column: 7}- range: {start: 255, end: 262}+ start:+ line: 10+ column: 17+ end:+ line: 11+ column: 7+ range:+ start: 255+ end: 262empty: true--- description: "CRLF line endings"+- description: CRLF line endingsinput: "CREATE TABLE t\r\n(a int);\r\nCREATE TABLE t1(b int);"result:- text: "CREATE TABLE t\r\n(a int);"baseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 9}- range: {start: 0, end: 24}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 9+ range:+ start: 0+ end: 24empty: false- text: "\r\nCREATE TABLE t1(b int);"baseline: 1- start: {line: 3, column: 1}- end: {line: 3, column: 24}- range: {start: 24, end: 49}+ start:+ line: 2+ column: 9+ end:+ line: 3+ column: 24+ range:+ start: 24+ end: 49empty: false--- description: "unicode in table names"- input: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);"+- description: unicode in table names+ input: "SELECT * FROM \u8868\u540D; INSERT INTO \u8868 VALUES (1);"result:- - text: "SELECT * FROM 表名;"+ - text: "SELECT * FROM \u8868\u540D;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 18}- range: {start: 0, end: 21}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 18+ range:+ start: 0+ end: 21empty: false- - text: " INSERT INTO 表 VALUES (1);"+ - text: " INSERT INTO \u8868 VALUES (1);"baseline: 0- start: {line: 1, column: 19}- end: {line: 1, column: 44}- range: {start: 21, end: 49}+ start:+ line: 1+ column: 18+ end:+ line: 1+ column: 44+ range:+ start: 21+ end: 49empty: falsediff --git a/backend/plugin/parser/partiql/completion.go b/backend/plugin/parser/partiql/completion.goindex 55ce5f8073..d0b033def7 100644--- a/backend/plugin/parser/partiql/completion.go+++ b/backend/plugin/parser/partiql/completion.go@@ -569,8 +569,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.if caretLine == previousSQLEndLine {// The caret is in the same line as the last line of the previous SQL statement.- // We need to adjust the caret offset.- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)+ newCaretOffset = caretOffset - previousSQLEndColumn + 1}break}diff --git a/backend/plugin/parser/partiql/split.go b/backend/plugin/parser/partiql/split.goindex b7f59a73d1..c5fc67ecd6 100644--- a/backend/plugin/parser/partiql/split.go+++ b/backend/plugin/parser/partiql/split.go@@ -18,5 +18,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()- return base.SplitSQLByLexer(stream, parser.PartiQLLexerCOLON_SEMI, statement)+ return base.SplitSQLByLexer(stream, parser.PartiQLLexerCOLON_SEMI)}diff --git a/backend/plugin/parser/partiql/test-data/test_split.yaml b/backend/plugin/parser/partiql/test-data/test_split.yamlindex 3962b333cb..477ea57888 100644--- a/backend/plugin/parser/partiql/test-data/test_split.yaml+++ b/backend/plugin/parser/partiql/test-data/test_split.yaml@@ -1,33 +1,120 @@-- description: "multiple DML statements"+- description: multiple DML statementsinput: "DELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money';\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money'};\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money02'};\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money03'};\n\tDELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money02'"result:- text: "DELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money';"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 71}- range: {start: 0, end: 70}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 71+ range:+ start: 0+ end: 70empty: false- text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money'};"baseline: 0- start: {line: 2, column: 2}- end: {line: 2, column: 131}- range: {start: 70, end: 201}+ start:+ line: 1+ column: 71+ end:+ line: 2+ column: 131+ range:+ start: 70+ end: 201empty: false- text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money02'};"baseline: 1- start: {line: 3, column: 2}- end: {line: 3, column: 133}- range: {start: 201, end: 334}+ start:+ line: 2+ column: 131+ end:+ line: 3+ column: 133+ range:+ start: 201+ end: 334empty: false- text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money03'};"baseline: 2- start: {line: 4, column: 2}- end: {line: 4, column: 133}- range: {start: 334, end: 467}+ start:+ line: 3+ column: 133+ end:+ line: 4+ column: 133+ range:+ start: 334+ end: 467empty: false- text: "\n\tDELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money02'"baseline: 3- start: {line: 5, column: 2}- end: {line: 5, column: 73}- range: {start: 467, end: 540}+ start:+ line: 4+ column: 133+ end:+ line: 5+ column: 73+ range:+ start: 467+ end: 540+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.goindex 512c350b6f..44cffcb7f6 100644--- a/backend/plugin/parser/pg/split.go+++ b/backend/plugin/parser/pg/split.go@@ -70,9 +70,8 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlstmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -85,7 +84,10 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),})byteOffset += stmtByteLength@@ -97,9 +99,8 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlstmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -112,7 +113,10 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlint32(tokens[eofPos-1].GetColumn()),tokens[eofPos-1].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:eofPos], parser.PostgreSQLParserSEMI),})}@@ -207,9 +211,8 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.StastmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -222,7 +225,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Staint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),})byteOffset += stmtByteLength@@ -234,9 +240,8 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.StastmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -249,7 +254,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Staint32(tokens[eofPos-1].GetColumn()),tokens[eofPos-1].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:eofPos], parser.PostgreSQLParserSEMI),})}diff --git a/backend/plugin/parser/pg/test-data/test_split.yaml b/backend/plugin/parser/pg/test-data/test_split.yamlindex 92e96fec00..e3f0a9b6c7 100644--- a/backend/plugin/parser/pg/test-data/test_split.yaml+++ b/backend/plugin/parser/pg/test-data/test_split.yaml@@ -1,357 +1,624 @@-- description: "single statement with semicolon"+- description: single statement with semicoloninput: "SELECT * FROM users;"result:- text: "SELECT * FROM users;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 21}- range: {start: 0, end: 20}- empty: false--- description: "single statement without semicolon"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 21+ range:+ start: 0+ end: 20+ empty: false+- description: single statement without semicoloninput: "SELECT * FROM users"result:- text: "SELECT * FROM users"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 20}- range: {start: 0, end: 19}- empty: false--- description: "multiple statements same line"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 20+ range:+ start: 0+ end: 19+ empty: false+- description: multiple statements same lineinput: "SELECT 1; SELECT 2;"result:- text: "SELECT 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- text: " SELECT 2;"baseline: 0- start: {line: 1, column: 11}- end: {line: 1, column: 20}- range: {start: 9, end: 19}- empty: false--- description: "leading whitespace preserved"+ start:+ line: 1+ column: 10+ end:+ line: 1+ column: 20+ range:+ start: 9+ end: 19+ empty: false+- description: leading whitespace preservedinput: " SELECT 1; SELECT 2;"result:- text: " SELECT 1;"baseline: 0- start: {line: 1, column: 5}- end: {line: 1, column: 14}- range: {start: 0, end: 13}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 14+ range:+ start: 0+ end: 13empty: false- text: " SELECT 2;"baseline: 0- start: {line: 1, column: 16}- end: {line: 1, column: 25}- range: {start: 13, end: 24}- empty: false--- description: "unicode in table name and values"+ start:+ line: 1+ column: 14+ end:+ line: 1+ column: 25+ range:+ start: 13+ end: 24+ empty: false+- description: unicode in table name and valuesinput: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);"result:- text: "SELECT * FROM 表名;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 18}- range: {start: 0, end: 21}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 18+ range:+ start: 0+ end: 21empty: false- text: " INSERT INTO 表 VALUES (1);"baseline: 0- start: {line: 1, column: 19}- end: {line: 1, column: 44}- range: {start: 21, end: 49}- empty: false--- description: "simple dollar-quoted string"+ start:+ line: 1+ column: 18+ end:+ line: 1+ column: 44+ range:+ start: 21+ end: 49+ empty: false+- description: simple dollar-quoted stringinput: "SELECT $$hello; world$$;"result:- text: "SELECT $$hello; world$$;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 25}- range: {start: 0, end: 24}- empty: false--- description: "tagged dollar-quoted string"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 25+ range:+ start: 0+ end: 24+ empty: false+- description: tagged dollar-quoted stringinput: "SELECT $tag$hello; world$tag$;"result:- text: "SELECT $tag$hello; world$tag$;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 31}- range: {start: 0, end: 30}- empty: false--- description: "dollar-quote does not split on internal semicolon"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 31+ range:+ start: 0+ end: 30+ empty: false+- description: dollar-quote does not split on internal semicoloninput: "SELECT $$a;b$$; SELECT 2;"result:- text: "SELECT $$a;b$$;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 16}- range: {start: 0, end: 15}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 16+ range:+ start: 0+ end: 15empty: false- text: " SELECT 2;"baseline: 0- start: {line: 1, column: 17}- end: {line: 1, column: 26}- range: {start: 15, end: 25}- empty: false--- description: "CRLF line endings"+ start:+ line: 1+ column: 16+ end:+ line: 1+ column: 26+ range:+ start: 15+ end: 25+ empty: false+- description: CRLF line endingsinput: "SELECT 1;\r\nSELECT 2;"result:- text: "SELECT 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- text: "\r\nSELECT 2;"baseline: 0- start: {line: 2, column: 1}- end: {line: 2, column: 10}- range: {start: 9, end: 20}- empty: false--- description: "multi-line with CRLF"+ start:+ line: 1+ column: 10+ end:+ line: 2+ column: 10+ range:+ start: 9+ end: 20+ empty: false+- description: multi-line with CRLFinput: "CREATE TABLE t\r\n(a int);\r\nCREATE TABLE t1(b int);"result:- text: "CREATE TABLE t\r\n(a int);"baseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 9}- range: {start: 0, end: 24}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 9+ range:+ start: 0+ end: 24empty: false- text: "\r\nCREATE TABLE t1(b int);"baseline: 1- start: {line: 3, column: 1}- end: {line: 3, column: 24}- range: {start: 24, end: 49}- empty: false--- description: "procedure with dollar-quoted body"- input: |-- CREATE PROCEDURE insert_data(a varchar(50))- LANGUAGE SQL- AS $$- INSERT INTO tbl VALUES ('test;value');- $$;- SELECT 1;+ start:+ line: 2+ column: 9+ end:+ line: 3+ column: 24+ range:+ start: 24+ end: 49+ empty: false+- description: procedure with dollar-quoted body+ input: "CREATE PROCEDURE insert_data(a varchar(50))\nLANGUAGE SQL\nAS $$\n INSERT INTO tbl VALUES ('test;value');\n$$;\nSELECT 1;"result:- text: "CREATE PROCEDURE insert_data(a varchar(50))\nLANGUAGE SQL\nAS $$\n INSERT INTO tbl VALUES ('test;value');\n$$;"baseline: 0- start: {line: 1, column: 1}- end: {line: 5, column: 4}- range: {start: 0, end: 107}+ start:+ line: 1+ column: 1+ end:+ line: 5+ column: 4+ range:+ start: 0+ end: 107empty: false- text: "\nSELECT 1;"baseline: 4- start: {line: 6, column: 1}- end: {line: 6, column: 10}- range: {start: 107, end: 117}- empty: false--- description: "function with tagged dollar-quote"- input: |-- CREATE FUNCTION get_data()- RETURNS TEXT- AS $func$- SELECT 'value with $$ inside';- $func$;- SELECT 2;+ start:+ line: 5+ column: 4+ end:+ line: 6+ column: 10+ range:+ start: 107+ end: 117+ empty: false+- description: function with tagged dollar-quote+ input: "CREATE FUNCTION get_data()\nRETURNS TEXT\nAS $func$\n SELECT 'value with $$ inside';\n$func$;\nSELECT 2;"result:- text: "CREATE FUNCTION get_data()\nRETURNS TEXT\nAS $func$\n SELECT 'value with $$ inside';\n$func$;"baseline: 0- start: {line: 1, column: 1}- end: {line: 5, column: 8}- range: {start: 0, end: 90}+ start:+ line: 1+ column: 1+ end:+ line: 5+ column: 8+ range:+ start: 0+ end: 90empty: false- text: "\nSELECT 2;"baseline: 4- start: {line: 6, column: 1}- end: {line: 6, column: 10}- range: {start: 90, end: 100}- empty: false--- description: "comments only creates empty statement"+ start:+ line: 5+ column: 8+ end:+ line: 6+ column: 10+ range:+ start: 90+ end: 100+ empty: false+- description: comments only creates empty statementinput: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n\t\t\t"result:- text: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n\t\t\t"baseline: 0- start: {line: 3, column: 4}- end: {line: 3, column: 4}- range: {start: 0, end: 42}+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 4+ range:+ start: 0+ end: 42empty: true--- description: "statement followed by trailing comment"+- description: statement followed by trailing commentinput: "select * from t;\n\t\t\t/* sdfasdf */"result:- text: "select * from t;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 17}- range: {start: 0, end: 16}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 17+ range:+ start: 0+ end: 16empty: false- text: "\n\t\t\t/* sdfasdf */"baseline: 0- start: {line: 2, column: 17}- end: {line: 2, column: 17}- range: {start: 16, end: 33}+ start:+ line: 1+ column: 17+ end:+ line: 2+ column: 17+ range:+ start: 16+ end: 33empty: true--- description: "statement then empty comment statement then statement"+- description: statement then empty comment statement then statementinput: "select * from t;\n\t\t\t/* sdfasdf */;\n\t\t\tselect * from t;"result:- text: "select * from t;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 17}- range: {start: 0, end: 16}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 17+ range:+ start: 0+ end: 16empty: false- text: "\n\t\t\t/* sdfasdf */;"baseline: 0- start: {line: 2, column: 17}- end: {line: 2, column: 18}- range: {start: 16, end: 34}+ start:+ line: 1+ column: 17+ end:+ line: 2+ column: 18+ range:+ start: 16+ end: 34empty: true- text: "\n\t\t\tselect * from t;"baseline: 1- start: {line: 3, column: 4}- end: {line: 3, column: 20}- range: {start: 34, end: 54}- empty: false--- description: "double-quoted identifiers with special characters"+ start:+ line: 2+ column: 18+ end:+ line: 3+ column: 20+ range:+ start: 34+ end: 54+ empty: false+- description: double-quoted identifiers with special charactersinput: "CREATE TABLE \"tech_Book\"(id int, name varchar(255));\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"result:- text: "CREATE TABLE \"tech_Book\"(id int, name varchar(255));"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 53}- range: {start: 0, end: 52}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 53+ range:+ start: 0+ end: 52empty: false- text: "\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"baseline: 0- start: {line: 2, column: 7}- end: {line: 2, column: 83}- range: {start: 52, end: 135}- empty: false--- description: "block comment and inline comment with statement"- input: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"+ start:+ line: 1+ column: 53+ end:+ line: 2+ column: 83+ range:+ start: 52+ end: 135+ empty: false+- description: block comment and inline comment with statement+ input: "\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"result:- - text: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));"- baseline: 0- start: {line: 3, column: 7}- end: {line: 3, column: 79}- range: {start: 0, end: 112}+ - text: "\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 79+ range:+ start: 0+ end: 111empty: false- text: "\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');"- baseline: 2- start: {line: 5, column: 7}- end: {line: 5, column: 83}- range: {start: 112, end: 225}- empty: false--- description: "unterminated single quote"+ baseline: 1+ start:+ line: 2+ column: 79+ end:+ line: 4+ column: 83+ range:+ start: 111+ end: 224+ empty: false+- description: unterminated single quoteinput: "INSERT INTO t VALUES ('klajfas)"result:- text: "INSERT INTO t VALUES ('klajfas)"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 32}- range: {start: 0, end: 31}- empty: false--- description: "unterminated double quote"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 32+ range:+ start: 0+ end: 31+ empty: false+- description: unterminated double quoteinput: "INSERT INTO \"t VALUES ('klajfas)"result:- text: "INSERT INTO \"t VALUES ('klajfas)"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 33}- range: {start: 0, end: 32}- empty: false--- description: "unterminated block comment"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 33+ range:+ start: 0+ end: 32+ empty: false+- description: unterminated block commentinput: "/*INSERT INTO \"t VALUES ('klajfas)"result:- text: "/*INSERT INTO \"t VALUES ('klajfas)"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 35}- range: {start: 0, end: 34}- empty: false--- description: "unterminated dollar quote"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 35+ range:+ start: 0+ end: 34+ empty: false+- description: unterminated dollar quoteinput: "$$INSERT INTO \"t VALUES ('klajfas)"result:- text: "$$INSERT INTO \"t VALUES ('klajfas)"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 35}- range: {start: 0, end: 34}- empty: false--- description: "complex procedure with comments and escapes inside dollar-quote"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 35+ range:+ start: 0+ end: 34+ empty: false+- description: complex procedure with comments and escapes inside dollar-quoteinput: "CREATE PROCEDURE insert_data(a varchar(50), b varchar(50))\n\t\t\t\t\t\tLANGUAGE SQL\n\t\t\t\t\t\tAS $$\n\t\t\t\t\t\t/*this is the comment */\n\t\t\t\t\t\tINSERT /* inline comment */ INTO tbl VALUES ('lkjafd''lksjadlf;lk\\\\jasdflkasdf\"asdklf\\\\');\n\t\t\t\t\t\t-- this is the comment\n\t\t\t\t\t\tINSERT INTO tbl VALUES ('fasf_bkdjlfa');\n\t\t\t\t\t\t$$;\n\t\t\t\t\t\tCREATE TABLE t(a int);"result:- text: "CREATE PROCEDURE insert_data(a varchar(50), b varchar(50))\n\t\t\t\t\t\tLANGUAGE SQL\n\t\t\t\t\t\tAS $$\n\t\t\t\t\t\t/*this is the comment */\n\t\t\t\t\t\tINSERT /* inline comment */ INTO tbl VALUES ('lkjafd''lksjadlf;lk\\\\jasdflkasdf\"asdklf\\\\');\n\t\t\t\t\t\t-- this is the comment\n\t\t\t\t\t\tINSERT INTO tbl VALUES ('fasf_bkdjlfa');\n\t\t\t\t\t\t$$;"baseline: 0- start: {line: 1, column: 1}- end: {line: 8, column: 10}- range: {start: 0, end: 303}+ start:+ line: 1+ column: 1+ end:+ line: 8+ column: 10+ range:+ start: 0+ end: 303empty: false- text: "\n\t\t\t\t\t\tCREATE TABLE t(a int);"baseline: 7- start: {line: 9, column: 7}- end: {line: 9, column: 29}- range: {start: 303, end: 332}- empty: false--- description: "procedure with tagged dollar-quote containing nested dollar signs"+ start:+ line: 8+ column: 10+ end:+ line: 9+ column: 29+ range:+ start: 303+ end: 332+ empty: false+- description: procedure with tagged dollar-quote containing nested dollar signsinput: "CREATE PROCEDURE insert_data(a varchar(50), b varchar(50))\n\t\t\t\t\t\tLANGUAGE SQL\n\t\t\t\t\t\tAS $tag_name$\n\t\t\t\t\t\t/*this is the comment */\n\t\t\t\t\t\tINSERT /* inline comment */ INTO tbl VALUES ('lkjafd''lksjadlf;lkjasdflkasdf\"asdklf');\n\t\t\t\t\t\t-- this is the comment\n\t\t\t\t\t\tINSERT INTO tbl VALUES ('fasf_bkdjlfa');\n\t\t\t\t\t\t$tag_name$;\n\t\t\t\t\t\tCREATE TABLE t(a int);"result:- text: "CREATE PROCEDURE insert_data(a varchar(50), b varchar(50))\n\t\t\t\t\t\tLANGUAGE SQL\n\t\t\t\t\t\tAS $tag_name$\n\t\t\t\t\t\t/*this is the comment */\n\t\t\t\t\t\tINSERT /* inline comment */ INTO tbl VALUES ('lkjafd''lksjadlf;lkjasdflkasdf\"asdklf');\n\t\t\t\t\t\t-- this is the comment\n\t\t\t\t\t\tINSERT INTO tbl VALUES ('fasf_bkdjlfa');\n\t\t\t\t\t\t$tag_name$;"baseline: 0- start: {line: 1, column: 1}- end: {line: 8, column: 18}- range: {start: 0, end: 315}+ start:+ line: 1+ column: 1+ end:+ line: 8+ column: 18+ range:+ start: 0+ end: 315empty: false- text: "\n\t\t\t\t\t\tCREATE TABLE t(a int);"baseline: 7- start: {line: 9, column: 7}- end: {line: 9, column: 29}- range: {start: 315, end: 344}- empty: false--- description: "multiple statements with leading whitespace and no trailing semicolon"+ start:+ line: 8+ column: 18+ end:+ line: 9+ column: 29+ range:+ start: 315+ end: 344+ empty: false+- description: multiple statements with leading whitespace and no trailing semicoloninput: " CREATE TABLE t(a int); CREATE TABLE t1(a int)"result:- text: " CREATE TABLE t(a int);"baseline: 0- start: {line: 1, column: 5}- end: {line: 1, column: 27}- range: {start: 0, end: 26}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 27+ range:+ start: 0+ end: 26empty: false- text: " CREATE TABLE t1(a int)"baseline: 0- start: {line: 1, column: 28}- end: {line: 1, column: 50}- range: {start: 26, end: 49}- empty: false--- description: "complex INSERT with escaped strings and special characters"+ start:+ line: 1+ column: 27+ end:+ line: 1+ column: 50+ range:+ start: 26+ end: 49+ empty: false+- description: "position semantic: leading newlines"+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: "position semantic: leading spaces and newlines"+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: "position semantic: multi-statement with leading whitespace"+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: "SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22+ empty: false+- description: complex INSERT with escaped strings and special charactersinput: "INSERT INTO \"public\".\"table\"(\"id\",\"content\")\n\t\t\tVALUES\n\t\t\t(12,'table column name () { :xna,sydfn,,kasdfyn;}; /////test string/// 0'),\n\t\t\t(133,'knuandfan public table id'';create table t(a int, b int);set @text=''\\\\\\\\kdaminxkljasdfiebkla.unkonwn''+''abcdef.xyz\\\\''; local xxxyy.abcddd.mysql @text;------- '),\n\t\t\t(1444,'table t xyz abc a''a\\\\\\\\\\\\\\\\''b\"c>?>xxxxxx%}}%%>c<[[?${12344556778990{%}}cake\\\\');"result:- text: "INSERT INTO \"public\".\"table\"(\"id\",\"content\")\n\t\t\tVALUES\n\t\t\t(12,'table column name () { :xna,sydfn,,kasdfyn;}; /////test string/// 0'),\n\t\t\t(133,'knuandfan public table id'';create table t(a int, b int);set @text=''\\\\\\\\kdaminxkljasdfiebkla.unkonwn''+''abcdef.xyz\\\\''; local xxxyy.abcddd.mysql @text;------- '),\n\t\t\t(1444,'table t xyz abc a''a\\\\\\\\\\\\\\\\''b\"c>?>xxxxxx%}}%%>c<[[?${12344556778990{%}}cake\\\\');"baseline: 0- start: {line: 1, column: 1}- end: {line: 5, column: 93}- range: {start: 0, end: 400}+ start:+ line: 1+ column: 1+ end:+ line: 5+ column: 93+ range:+ start: 0+ end: 400empty: falsediff --git a/backend/plugin/parser/plsql/completion.go b/backend/plugin/parser/plsql/completion.goindex a74661dd0a..96bc115eb1 100644--- a/backend/plugin/parser/plsql/completion.go+++ b/backend/plugin/parser/plsql/completion.go@@ -1171,8 +1171,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.if caretLine == previousSQLEndLine {// The caret is in the same line as the last line of the previous SQL statement.- // We need to adjust the caret offset.- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)+ newCaretOffset = caretOffset - previousSQLEndColumn + 1}break}diff --git a/backend/plugin/parser/plsql/plsql.go b/backend/plugin/parser/plsql/plsql.goindex e1922732c4..27f839a21f 100644--- a/backend/plugin/parser/plsql/plsql.go+++ b/backend/plugin/parser/plsql/plsql.go@@ -119,22 +119,40 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {// Iterate through children in order to preserve statement ordering and re-parse each onevar result []*base.ANTLRAST+ prevStopTokenIndex := -1for _, child := range sqlScript.GetChildren() {var stmtText stringvar stmtBaseLine int+ var startToken, stopToken antlr.Token// Type assert to get the specific statement typeif stmt, ok := child.(parser.IUnit_statementContext); ok {- stmtText = tokens.GetTextFromTokens(stmt.GetStart(), stmt.GetStop())- stmtBaseLine = stmt.GetStart().GetLine() - 1 // Convert to 0-based+ startToken = stmt.GetStart()+ stopToken = stmt.GetStop()} else if sqlPlusCmd, ok := child.(parser.ISql_plus_commandContext); ok {- stmtText = tokens.GetTextFromTokens(sqlPlusCmd.GetStart(), sqlPlusCmd.GetStop())- stmtBaseLine = sqlPlusCmd.GetStart().GetLine() - 1 // Convert to 0-based+ startToken = sqlPlusCmd.GetStart()+ stopToken = sqlPlusCmd.GetStop()} else {// Skip other node types (e.g., EOF)continue}+ // Calculate the leading whitespace/comments before this statement (like SplitSQL does)+ leadingContent := ""+ if startTokenIndex := startToken.GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {+ leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(startTokenIndex-1))+ }++ // Include leading whitespace in the statement text+ stmtText = leadingContent + tokens.GetTextFromTokens(startToken, stopToken)++ // stmtBaseLine is where the leading content starts (for re-parsing with correct offsets).+ // This ensures token positions in the re-parsed AST are correct when combined with SplitSQL's BaseLine.+ // Formula: first token's line - 1 (convert to 0-based) - number of newlines in leading content+ stmtBaseLine = startToken.GetLine() - 1 - strings.Count(leadingContent, "\n")++ prevStopTokenIndex = stopToken.GetTokenIndex()+// Skip empty statementsif strings.TrimSpace(stmtText) == "" || stmtText == ";" {continue@@ -146,6 +164,8 @@ func ParsePLSQL(sql string) ([]*base.ANTLRAST, error) {return nil, err}+ // StartPosition points to the first character of Text (including leading whitespace),+ // not the first token. This enables replacing BaseLine with Start.GetLine() - 1 in advisors.result = append(result, &base.ANTLRAST{StartPosition: &storepb.Position{Line: int32(stmtBaseLine) + 1},Tree: stmtTree,diff --git a/backend/plugin/parser/plsql/plsql_test.go b/backend/plugin/parser/plsql/plsql_test.goindex 6c17bbd58f..47695443e8 100644--- a/backend/plugin/parser/plsql/plsql_test.go+++ b/backend/plugin/parser/plsql/plsql_test.go@@ -61,7 +61,7 @@ func TestPLSQLParser_MultipleStatements(t *testing.T) {tests := []struct {statement stringexpectedCount int // Number of individual statements- expectedLines []int // BaseLine for each result+ expectedLines []int // StartPosition.Line - 1 for each result (0-based line of first character, including leading whitespace)}{{statement: "SELECT * FROM t1;",@@ -69,17 +69,19 @@ func TestPLSQLParser_MultipleStatements(t *testing.T) {expectedLines: []int{0},},{+ // Statement 2's first character is the newline at end of line 1statement: `SELECT * FROM t1;SELECT * FROM t2;`,expectedCount: 2,- expectedLines: []int{0, 1},+ expectedLines: []int{0, 0},},{+ // Statement 2's first char is newline at end of line 1, statement 3's first char is newline at end of line 2statement: `SELECT * FROM t1;SELECT * FROM t2;INSERT INTO t3 VALUES (1, 2);`,expectedCount: 3,- expectedLines: []int{0, 1, 2},+ expectedLines: []int{0, 0, 1},},}diff --git a/backend/plugin/parser/plsql/restore.go b/backend/plugin/parser/plsql/restore.goindex df8997e1de..e76da66dcb 100644--- a/backend/plugin/parser/plsql/restore.go+++ b/backend/plugin/parser/plsql/restore.go@@ -337,7 +337,8 @@ func extractSQL(statement string, backupItem *storepb.PriorBackupDetail_Item) (sresult = append(result, list[i].Text)}}- return strings.Join(result, ";\n"), nil+ // Statements include their leading whitespace and trailing semicolons.+ return strings.Join(result, ""), nil}func equalOrLess(a, b *storepb.Position) bool {diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.goindex f021b85b41..e6259259db 100644--- a/backend/plugin/parser/plsql/split.go+++ b/backend/plugin/parser/plsql/split.go@@ -1,15 +1,12 @@package plsqlimport (- "strings"-"github.com/antlr4-go/antlr/v4"parser "github.com/bytebase/parser/plsql""github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"- "github.com/bytebase/bytebase/backend/utils")func init() {@@ -35,44 +32,66 @@ func SplitSQL(statement string) ([]base.Statement, error) {prevStopTokenIndex := -1var result []base.Statementfor _, item := range tree.GetChildren() {+ // Skip past sql_plus_command (like "/" block terminator) to prevent it from being+ // included in the next statement's leadingContent.+ if sqlPlusCmd, ok := item.(parser.ISql_plus_commandContext); ok {+ // Calculate the leading whitespace/comments before this sql_plus_command+ leadingContent := ""+ if startTokenIndex := sqlPlusCmd.GetStart().GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {+ leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(sqlPlusCmd.GetStart().GetTokenIndex()-1))+ }+ // Skip past both the leading content and the command itself+ cmdText := tokens.GetTextFromTokens(sqlPlusCmd.GetStart(), sqlPlusCmd.GetStop())+ byteOffsetStart += len(leadingContent) + len(cmdText)+ prevStopTokenIndex = sqlPlusCmd.GetStop().GetTokenIndex()+ continue+ }+if stmt, ok := item.(parser.IUnit_statementContext); ok {text := ""var lastToken antlr.Token- // tokens looks like++ // Calculate the leading whitespace/comments before this statement+ leadingContent := ""if startTokenIndex := stmt.GetStart().GetTokenIndex(); startTokenIndex-1 >= 0 && prevStopTokenIndex+1 <= startTokenIndex-1 {- byteOffsetStart += len(tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(stmt.GetStart().GetTokenIndex()-1)))+ leadingContent = tokens.GetTextFromTokens(tokens.Get(prevStopTokenIndex+1), tokens.Get(stmt.GetStart().GetTokenIndex()-1))}- byteOffsetEnd := byteOffsetStart + len(tokens.GetTextFromTokens(stmt.GetStart(), stmt.GetStop()))- // The go-ora driver requires semicolon for anonymous block,- // but does not support semicolon for other statements.+ // The go-ora driver requires semicolon for anonymous blocks/procedures/functions,+ // but does NOT support semicolon for other statements (CREATE TABLE, SELECT, etc.).+ stopTokenIndex := stmt.GetStop().GetTokenIndex()if needSemicolon(stmt) {- lastToken = tokens.Get(stmt.GetStop().GetTokenIndex())- text = tokens.GetTextFromTokens(stmt.GetStart(), lastToken)+ // For procedures/functions/anonymous blocks: include semicolon if present, add if missing+ lastToken = tokens.Get(stopTokenIndex)+ text = leadingContent + tokens.GetTextFromTokens(stmt.GetStart(), lastToken)if lastToken.GetTokenType() != parser.PlSqlParserSEMICOLON {text += ";"}} else {- stopIndex := stmt.GetStop().GetTokenIndex()+ // For regular statements: EXCLUDE the semicolon (go-ora doesn't support it)if stmt.GetStop().GetTokenType() == parser.PlSqlParserSEMICOLON {- stopIndex--+ stopTokenIndex--}- lastToken = tokens.Get(stopIndex)- text = tokens.GetTextFromTokens(stmt.GetStart(), lastToken)- text = strings.TrimRightFunc(text, utils.IsSpaceOrSemicolon)+ lastToken = tokens.Get(stopTokenIndex)+ text = leadingContent + tokens.GetTextFromTokens(stmt.GetStart(), lastToken)}+ // Calculate byte offsets using lastToken (which includes semicolon if present)+ // byteOffsetStart is where the previous statement ended (including any leading whitespace)+ tokenByteOffset := byteOffsetStart + len(leadingContent)+ byteOffsetEnd := tokenByteOffset + len(tokens.GetTextFromTokens(stmt.GetStart(), lastToken))++ // Calculate start position based on byteOffsetStart (including leading whitespace)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffsetStart)+result = append(result, base.Statement{Text: text,// BaseLine is 0-based line offset of this statement in the original SQL- BaseLine: stmt.GetStart().GetLine() - 1,- Start: common.ConvertANTLRPositionToPosition(- &common.ANTLRPosition{- Line: int32(stmt.GetStart().GetLine()),- Column: int32(stmt.GetStart().GetColumn()),- },- statement,- ),+ BaseLine: startLine,+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },End: common.ConvertANTLRTokenToExclusiveEndPosition(int32(lastToken.GetLine()),int32(lastToken.GetColumn()),@@ -85,7 +104,15 @@ func SplitSQL(statement string) ([]base.Statement, error) {},})byteOffsetStart = byteOffsetEnd+ // Set prevStopTokenIndex to the last token we want to "consume" for this statement.+ // For statements where the semicolon is a separator (not part of the statement parse tree),+ // we need to skip past the semicolon so it's not included in the next statement's leadingContent.prevStopTokenIndex = stmt.GetStop().GetTokenIndex()+ if nextIdx := prevStopTokenIndex + 1; nextIdx < len(tokens.GetAllTokens()) {+ if nextToken := tokens.Get(nextIdx); nextToken.GetTokenType() == parser.PlSqlParserSEMICOLON {+ prevStopTokenIndex = nextIdx+ }+ }}}return result, nildiff --git a/backend/plugin/parser/plsql/test-data/test_restore.yaml b/backend/plugin/parser/plsql/test-data/test_restore.yamlindex d0542cffdc..cf86b096fc 100644--- a/backend/plugin/parser/plsql/test-data/test_restore.yaml+++ b/backend/plugin/parser/plsql/test-data/test_restore.yaml@@ -13,12 +13,12 @@result: |-/*Original SQL:- UPDATE T_GENERATED SET b = 1 WHERE a = 1;- UPDATE T_GENERATED SET b = 2 WHERE a = 2;- UPDATE T_GENERATED SET b = 3 WHERE a = 3;- UPDATE T_GENERATED SET b = 4 WHERE a = 4;- UPDATE T_GENERATED SET b = 5 WHERE a = 5;- UPDATE T_GENERATED SET b = 6 WHERE a = 6;+ UPDATE T_GENERATED SET b = 1 WHERE a = 1+ UPDATE T_GENERATED SET b = 2 WHERE a = 2+ UPDATE T_GENERATED SET b = 3 WHERE a = 3+ UPDATE T_GENERATED SET b = 4 WHERE a = 4+ UPDATE T_GENERATED SET b = 5 WHERE a = 5+ UPDATE T_GENERATED SET b = 6 WHERE a = 6UPDATE T_GENERATED SET b = 7 WHERE a = 7*/MERGE INTO "DB"."T_GENERATED" t@@ -43,12 +43,12 @@result: |-/*Original SQL:- UPDATE T_GENERATED SET a = 1 WHERE b = 1;- UPDATE T_GENERATED SET a = 2 WHERE b = 2;- UPDATE T_GENERATED SET a = 3 WHERE b = 3;- UPDATE T_GENERATED SET a = 4 WHERE b = 4;- UPDATE T_GENERATED SET a = 5 WHERE b = 5;- UPDATE T_GENERATED SET a = 6 WHERE b = 6;+ UPDATE T_GENERATED SET a = 1 WHERE b = 1+ UPDATE T_GENERATED SET a = 2 WHERE b = 2+ UPDATE T_GENERATED SET a = 3 WHERE b = 3+ UPDATE T_GENERATED SET a = 4 WHERE b = 4+ UPDATE T_GENERATED SET a = 5 WHERE b = 5+ UPDATE T_GENERATED SET a = 6 WHERE b = 6UPDATE T_GENERATED SET a = 7 WHERE b = 7*/MERGE INTO "DB"."T_GENERATED" tdiff --git a/backend/plugin/parser/plsql/test-data/test_split.yaml b/backend/plugin/parser/plsql/test-data/test_split.yamlindex 5131fb9e2d..673468e353 100644--- a/backend/plugin/parser/plsql/test-data/test_split.yaml+++ b/backend/plugin/parser/plsql/test-data/test_split.yaml@@ -1,121 +1,297 @@-- description: "multiple SELECT statements"- input: "SELECT * FROM t1; SELECT * FROM t2"+- description: multiple SELECT statements+ input: SELECT * FROM t1; SELECT * FROM t2result:- - text: "SELECT * FROM t1"+ - text: SELECT * FROM t1baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 17}- range: {start: 0, end: 16}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 17+ range:+ start: 0+ end: 16empty: false- - text: "SELECT * FROM t2"+ - text: " SELECT * FROM t2"baseline: 0- start: {line: 1, column: 19}- end: {line: 1, column: 35}- range: {start: 18, end: 34}+ start:+ line: 1+ column: 17+ end:+ line: 1+ column: 35+ range:+ start: 16+ end: 33empty: false--- description: "multiple statements with newlines"+- description: multiple statements with newlinesinput: "\n\t\t\t\tselect * from t;\n\t\t\t\tcreate table table$1 (id int)\n\t\t\t"result:- - text: "select * from t"+ - text: "\n\t\t\t\tselect * from t"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 20+ range:+ start: 0+ end: 20+ empty: false+ - text: "\n\t\t\t\tcreate table table$1 (id int)"baseline: 1- start: {line: 2, column: 5}- end: {line: 2, column: 20}- range: {start: 5, end: 20}- empty: false- - text: "create table table$1 (id int)"- baseline: 2- start: {line: 3, column: 5}- end: {line: 3, column: 34}- range: {start: 26, end: 56}- empty: false--- description: "procedure with forward slash separator"- input: "CREATE OR REPLACE PROCEDURE test_proc IS\nBEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Hello');\nEND;\n/"+ start:+ line: 2+ column: 20+ end:+ line: 3+ column: 34+ range:+ start: 20+ end: 54+ empty: false+- description: procedure with forward slash separator+ input: |-+ CREATE OR REPLACE PROCEDURE test_proc IS+ BEGIN+ CALL DBMS_OUTPUT.PUT_LINE('Hello');+ END;+ /result:- - text: "CREATE OR REPLACE PROCEDURE test_proc IS\nBEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Hello');\nEND;"+ - text: |-+ CREATE OR REPLACE PROCEDURE test_proc IS+ BEGIN+ CALL DBMS_OUTPUT.PUT_LINE('Hello');+ END;baseline: 0- start: {line: 1, column: 1}- end: {line: 4, column: 5}- range: {start: 0, end: 91}+ start:+ line: 1+ column: 1+ end:+ line: 4+ column: 5+ range:+ start: 0+ end: 91empty: false--- description: "two procedures with forward slash separator"- input: "CREATE OR REPLACE PROCEDURE proc1 IS\nBEGIN\n NULL;\nEND;\n/\nCREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;\n/"+- description: two procedures with forward slash separator+ input: |-+ CREATE OR REPLACE PROCEDURE proc1 IS+ BEGIN+ NULL;+ END;+ /+ CREATE OR REPLACE PROCEDURE proc2 IS+ BEGIN+ NULL;+ END;+ /result:- - text: "CREATE OR REPLACE PROCEDURE proc1 IS\nBEGIN\n NULL;\nEND;"+ - text: |-+ CREATE OR REPLACE PROCEDURE proc1 IS+ BEGIN+ NULL;+ END;baseline: 0- start: {line: 1, column: 1}- end: {line: 4, column: 5}- range: {start: 0, end: 57}- empty: false- - text: "CREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;"- baseline: 5- start: {line: 6, column: 1}- end: {line: 9, column: 5}- range: {start: 60, end: 117}- empty: false--- description: "SELECT statements separated by forward slash"- input: "SELECT * FROM t1;\n/\nSELECT * FROM t2;"+ start:+ line: 1+ column: 1+ end:+ line: 4+ column: 5+ range:+ start: 0+ end: 57+ empty: false+ - text: "\nCREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;"+ baseline: 4+ start:+ line: 5+ column: 2+ end:+ line: 9+ column: 5+ range:+ start: 59+ end: 117+ empty: false+- description: SELECT statements separated by forward slash+ input: |-+ SELECT * FROM t1;+ /+ SELECT * FROM t2;result:- - text: "SELECT * FROM t1"+ - text: SELECT * FROM t1baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 17}- range: {start: 0, end: 16}- empty: false- - text: "SELECT * FROM t2"- baseline: 2- start: {line: 3, column: 1}- end: {line: 3, column: 17}- range: {start: 20, end: 36}- empty: false--- description: "anonymous block with forward slash"- input: "BEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Test');\nEND;\n/"+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 17+ range:+ start: 0+ end: 16+ empty: false+ - text: "\nSELECT * FROM t2"+ baseline: 1+ start:+ line: 2+ column: 1+ end:+ line: 3+ column: 17+ range:+ start: 18+ end: 35+ empty: false+- description: anonymous block with forward slash+ input: |-+ BEGIN+ CALL DBMS_OUTPUT.PUT_LINE('Test');+ END;+ /result:- - text: "BEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Test');\nEND;"+ - text: |-+ BEGIN+ CALL DBMS_OUTPUT.PUT_LINE('Test');+ END;baseline: 0- start: {line: 1, column: 1}- end: {line: 3, column: 5}- range: {start: 0, end: 49}+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 5+ range:+ start: 0+ end: 49empty: false--- description: "ALTER TABLE with PARTITION"- input: "ALTER TABLE DATA.TEST\nMODIFY PARTITION BY RANGE (TXN_DATE)\nINTERVAL (NUMTODSINTERVAL(1, 'DAY'))\n(\n\tPARTITION TEST_PO VALUES LESS THAN (\n\t\tTO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')\n\t)\n)\nONLINE;"+- description: ALTER TABLE with PARTITION+ input: |-+ ALTER TABLE DATA.TEST+ MODIFY PARTITION BY RANGE (TXN_DATE)+ INTERVAL (NUMTODSINTERVAL(1, 'DAY'))+ (+ PARTITION TEST_PO VALUES LESS THAN (+ TO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')+ )+ )+ ONLINE;result:- - text: "ALTER TABLE DATA.TEST\nMODIFY PARTITION BY RANGE (TXN_DATE)\nINTERVAL (NUMTODSINTERVAL(1, 'DAY'))\n(\n\tPARTITION TEST_PO VALUES LESS THAN (\n\t\tTO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')\n\t)\n)\nONLINE"+ - text: |-+ ALTER TABLE DATA.TEST+ MODIFY PARTITION BY RANGE (TXN_DATE)+ INTERVAL (NUMTODSINTERVAL(1, 'DAY'))+ (+ PARTITION TEST_PO VALUES LESS THAN (+ TO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')+ )+ )+ ONLINEbaseline: 0- start: {line: 1, column: 1}- end: {line: 9, column: 7}- range: {start: 0, end: 233}+ start:+ line: 1+ column: 1+ end:+ line: 9+ column: 7+ range:+ start: 0+ end: 232empty: false--- description: "CALL procedure"- input: "CALL my_procedure()"+- description: CALL procedure+ input: CALL my_procedure()result:- - text: "CALL my_procedure()"+ - text: CALL my_procedure()baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 20}- range: {start: 0, end: 19}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 20+ range:+ start: 0+ end: 19empty: false--- description: "DROP TABLESPACE with CASCADE CONSTRAINTS"- input: "DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS"+- description: DROP TABLESPACE with CASCADE CONSTRAINTS+ input: DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTSresult:- - text: "DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS"+ - text: DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTSbaseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 59}- range: {start: 0, end: 58}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 59+ range:+ start: 0+ end: 58empty: false--- description: "DROP TABLESPACE CASCADE alone should error"- input: "DROP TABLESPACE xxx CASCADE"+- description: DROP TABLESPACE CASCADE alone should error+ input: DROP TABLESPACE xxx CASCADEerror: "Syntax error at line 1:28 \nrelated text: DROP TABLESPACE xxx CASCADE;"--- description: "DROP TABLESPACE then CASCADE should error"- input: "DROP TABLESPACE xxx; CASCADE"+- description: DROP TABLESPACE then CASCADE should error+ input: DROP TABLESPACE xxx; CASCADEerror: "Syntax error at line 1:29 \nrelated text: DROP TABLESPACE xxx; CASCADE;"+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1 FROM dual"+ result:+ - text: "\n\nSELECT 1 FROM dual"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 19+ range:+ start: 0+ end: 20+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1 FROM dual"+ result:+ - text: "\n \n SELECT 1 FROM dual"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 21+ range:+ start: 0+ end: 24+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1 FROM dual;\n\n SELECT 2 FROM dual"+ result:+ - text: SELECT 1 FROM dual+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 19+ range:+ start: 0+ end: 18+ empty: false+ - text: "\n\n SELECT 2 FROM dual"+ baseline: 0+ start:+ line: 1+ column: 19+ end:+ line: 3+ column: 21+ range:+ start: 18+ end: 40+ empty: falsediff --git a/backend/plugin/parser/redshift/completion.go b/backend/plugin/parser/redshift/completion.goindex e92097e1f1..e40f18921c 100644--- a/backend/plugin/parser/redshift/completion.go+++ b/backend/plugin/parser/redshift/completion.go@@ -1287,8 +1287,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.if caretLine == previousSQLEndLine {// The caret is in the same line as the last line of the previous SQL statement.- // We need to adjust the caret offset.- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)+ newCaretOffset = caretOffset - previousSQLEndColumn + 1}break}diff --git a/backend/plugin/parser/redshift/split.go b/backend/plugin/parser/redshift/split.goindex d16d3ecbde..0c1fb35bfc 100644--- a/backend/plugin/parser/redshift/split.go+++ b/backend/plugin/parser/redshift/split.go@@ -71,9 +71,8 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -86,7 +85,10 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.int32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:pos+1], parser.RedshiftParserSEMI),})byteOffset += stmtByteLength@@ -98,9 +100,8 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -113,7 +114,10 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.int32(tokens[eofPos-1].GetColumn()),tokens[eofPos-1].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:eofPos], parser.RedshiftParserSEMI),})}@@ -207,9 +211,8 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.StastmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -222,7 +225,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Staint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:pos+1], parser.RedshiftParserSEMI),})byteOffset += stmtByteLength@@ -234,9 +240,8 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.StastmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])- // From antlr4, the line is ONE based, and the column is ZERO based.- // So we should minus 1 for the line.+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -249,7 +254,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Staint32(tokens[eofPos-1].GetColumn()),tokens[eofPos-1].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:eofPos], parser.RedshiftParserSEMI),})}diff --git a/backend/plugin/parser/redshift/test-data/test_split.yaml b/backend/plugin/parser/redshift/test-data/test_split.yamlindex e2ef44caae..8ac26bfca8 100644--- a/backend/plugin/parser/redshift/test-data/test_split.yaml+++ b/backend/plugin/parser/redshift/test-data/test_split.yaml@@ -1,51 +1,141 @@-- description: "simple single statement"- input: "SELECT 1;"+- description: simple single statement+ input: SELECT 1;result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false--- description: "multiple statements"- input: "SELECT 1; SELECT 2;"+- description: multiple statements+ input: SELECT 1; SELECT 2;result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- - text: " SELECT 2;"+ - text: ' SELECT 2;'baseline: 0- start: {line: 1, column: 11}- end: {line: 1, column: 20}- range: {start: 9, end: 19}+ start:+ line: 1+ column: 10+ end:+ line: 1+ column: 20+ range:+ start: 9+ end: 19empty: false--- description: "multiple statements on multiple lines"+- description: multiple statements on multiple linesinput: "SELECT 1;\nSELECT 2;"result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- text: "\nSELECT 2;"baseline: 0- start: {line: 2, column: 1}- end: {line: 2, column: 10}- range: {start: 9, end: 19}+ start:+ line: 1+ column: 10+ end:+ line: 2+ column: 10+ range:+ start: 9+ end: 19empty: false--- description: "multi-line statement"+- description: multi-line statementinput: "SELECT\n 1;"result:- text: "SELECT\n 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 5}- range: {start: 0, end: 11}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 5+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/snowflake/split.go b/backend/plugin/parser/snowflake/split.goindex 88e01e96f7..5fbb47de13 100644--- a/backend/plugin/parser/snowflake/split.go+++ b/backend/plugin/parser/snowflake/split.go@@ -18,5 +18,5 @@ func SplitSQL(statement string) ([]base.Statement, error) {stream := antlr.NewCommonTokenStream(lexer, antlr.TokenDefaultChannel)stream.Fill()- return base.SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI, statement)+ return base.SplitSQLByLexer(stream, parser.SnowflakeLexerSEMI)}diff --git a/backend/plugin/parser/snowflake/test-data/test_split.yaml b/backend/plugin/parser/snowflake/test-data/test_split.yamlindex 195f6fee52..c68bc213de 100644--- a/backend/plugin/parser/snowflake/test-data/test_split.yaml+++ b/backend/plugin/parser/snowflake/test-data/test_split.yaml@@ -1,41 +1,126 @@-- description: "multiple statements on multiple lines"+- description: multiple statements on multiple linesinput: "SELECT * FROM users;\n\tSELECT * FROM orders;\n\tSELECT * FROM products;"result:- - text: "SELECT * FROM users;"+ - text: SELECT * FROM users;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 21}- range: {start: 0, end: 20}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 21+ range:+ start: 0+ end: 20empty: false- text: "\n\tSELECT * FROM orders;"baseline: 0- start: {line: 2, column: 2}- end: {line: 2, column: 23}- range: {start: 20, end: 43}+ start:+ line: 1+ column: 21+ end:+ line: 2+ column: 23+ range:+ start: 20+ end: 43empty: false- text: "\n\tSELECT * FROM products;"baseline: 1- start: {line: 3, column: 2}- end: {line: 3, column: 25}- range: {start: 43, end: 68}+ start:+ line: 2+ column: 23+ end:+ line: 3+ column: 25+ range:+ start: 43+ end: 68empty: false--- description: "single statement"- input: "SELECT * FROM users;"+- description: single statement+ input: SELECT * FROM users;result:- - text: "SELECT * FROM users;"+ - text: SELECT * FROM users;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 21}- range: {start: 0, end: 20}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 21+ range:+ start: 0+ end: 20empty: false--- description: "statement without semicolon"- input: "SELECT * FROM users"+- description: statement without semicolon+ input: SELECT * FROM usersresult:- - text: "SELECT * FROM users"+ - text: SELECT * FROM usersbaseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 20}- range: {start: 0, end: 19}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 20+ range:+ start: 0+ end: 19+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/spanner/split.go b/backend/plugin/parser/spanner/split.goindex a01347d955..c600d6582d 100644--- a/backend/plugin/parser/spanner/split.go+++ b/backend/plugin/parser/spanner/split.go@@ -92,7 +92,8 @@ func SplitSQL(statement string) ([]base.Statement, error) {stmtText := stream.GetTextFromTokens(tokens[start], tokens[endPos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : endPos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -105,7 +106,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {int32(tokens[endPos].GetColumn()),tokens[endPos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: base.IsEmpty(tokens[start:endPos+1], parser.GoogleSQLLexerSEMI_SYMBOL),})byteOffset += stmtByteLengthdiff --git a/backend/plugin/parser/spanner/test-data/test_split.yaml b/backend/plugin/parser/spanner/test-data/test_split.yamlindex 0043d41600..60ded7187e 100644--- a/backend/plugin/parser/spanner/test-data/test_split.yaml+++ b/backend/plugin/parser/spanner/test-data/test_split.yaml@@ -1,31 +1,111 @@-- description: "multiple statements on multiple lines"+- description: multiple statements on multiple linesinput: "SELECT * FROM users;\n\tSELECT * FROM orders;\n\tSELECT * FROM products;"result:- - text: "SELECT * FROM users;"+ - text: SELECT * FROM users;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 21}- range: {start: 0, end: 20}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 21+ range:+ start: 0+ end: 20empty: false- text: "\n\tSELECT * FROM orders;"baseline: 0- start: {line: 2, column: 2}- end: {line: 2, column: 23}- range: {start: 20, end: 43}+ start:+ line: 1+ column: 21+ end:+ line: 2+ column: 23+ range:+ start: 20+ end: 43empty: false- text: "\n\tSELECT * FROM products;"baseline: 1- start: {line: 3, column: 2}- end: {line: 3, column: 25}- range: {start: 43, end: 68}+ start:+ line: 2+ column: 23+ end:+ line: 3+ column: 25+ range:+ start: 43+ end: 68empty: false--- description: "single statement"- input: "SELECT * FROM users;"+- description: single statement+ input: SELECT * FROM users;result:- - text: "SELECT * FROM users;"+ - text: SELECT * FROM users;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 21}- range: {start: 0, end: 20}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 21+ range:+ start: 0+ end: 20+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/standard/split.go b/backend/plugin/parser/standard/split.goindex 43a76644e4..8313bb2cec 100644--- a/backend/plugin/parser/standard/split.go+++ b/backend/plugin/parser/standard/split.go@@ -24,22 +24,31 @@ func SplitSQL(statement string) ([]base.Statement, error) {var list []base.StatementbyteOffset := 0err := applyMultiStatements(strings.NewReader(statement), func(sql string) error {- // Find the start position of this SQL in the original statement- startPos := strings.Index(statement[byteOffset:], strings.TrimLeft(sql, "\n\t "))- if startPos != -1 {- startPos += byteOffset- } else {- startPos = byteOffset+ // sql may have trailing newline added by applyMultiStatements+ // Strip trailing whitespace for searching in the original statement+ sqlTrimmed := strings.TrimRight(sql, "\n\r\t ")++ // Find where the SQL content exists in the remaining statement+ sqlPos := strings.Index(statement[byteOffset:], sqlTrimmed)+ if sqlPos == -1 {+ sqlPos = 0}- endPos := startPos + len(sql)++ // startPos includes leading whitespace from where previous statement ended+ startPos := byteOffset+ // endPos is where the SQL content ends+ endPos := byteOffset + sqlPos + len(sqlTrimmed)++ // The actual text includes leading whitespace + SQL content+ text := statement[startPos:endPos]// Calculate line and column for Start position- startLine, startColumn := calculateLineAndColumn(statement, startPos)+ startLine, startColumn := base.CalculateLineAndColumn(statement, startPos)// Calculate line and column for End position- endLine, endColumn := calculateLineAndColumn(statement, endPos)+ endLine, endColumn := base.CalculateLineAndColumn(statement, endPos)list = append(list, base.Statement{- Text: sql,+ Text: text,BaseLine: startLine,Start: &storepb.Position{Line: int32(startLine + 1), // 1-based@@ -53,7 +62,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {Start: int32(startPos),End: int32(endPos),},- Empty: isEmptySQL(sql),+ Empty: isEmptySQL(text),})byteOffset = endPosreturn nil@@ -61,24 +70,6 @@ func SplitSQL(statement string) ([]base.Statement, error) {return list, err}-// calculateLineAndColumn calculates the 0-based line number and 0-based column (character offset)-// for a given byte offset in the statement.-func calculateLineAndColumn(statement string, byteOffset int) (line, column int) {- if byteOffset > len(statement) {- byteOffset = len(statement)- }- // Range over string iterates over runes (code points), not bytes- for _, r := range statement[:byteOffset] {- if r == '\n' {- line++- column = 0- } else {- column++- }- }- return line, column-}-// isEmptySQL checks if the SQL contains only whitespace and comments.func isEmptySQL(sql string) bool {trimmed := strings.TrimSpace(sql)@@ -173,7 +164,7 @@ func applyMultiStatements(sc io.Reader, f func(string) error) error {}if execute {s := sb.String()- s = strings.Trim(s, "\n\t ")+ // Don't trim - include leading whitespace in Text for position consistencyif s != "" {if err := f(s); err != nil {return errors.Wrapf(err, "execute query %q failed", s)@@ -184,7 +175,7 @@ func applyMultiStatements(sc io.Reader, f func(string) error) error {}// Apply the remaining content.s := sb.String()- s = strings.Trim(s, "\n\t ")+ // Don't trim - include leading whitespace in Text for position consistencyif s != "" {if err := f(s); err != nil {return errors.Wrapf(err, "execute query %q failed", s)diff --git a/backend/plugin/parser/standard/test-data/test_split.yaml b/backend/plugin/parser/standard/test-data/test_split.yamlindex 68a57e60d4..ed4a61600b 100644--- a/backend/plugin/parser/standard/test-data/test_split.yaml+++ b/backend/plugin/parser/standard/test-data/test_split.yaml@@ -1,81 +1,186 @@-- description: "simple single statement"+- description: simple single statementinput: "SELECT 1;"result:- text: "SELECT 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false--- description: "multi-line statement"+- description: multi-line statementinput: "SELECT\n 1;"result:- text: "SELECT\n 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 5}- range: {start: 0, end: 11}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 5+ range:+ start: 0+ end: 11empty: false--- description: "multiple statements"+- description: multiple statementsinput: "SELECT 1;\nSELECT 2;"result:- text: "SELECT 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- - text: "SELECT 2;"- baseline: 1- start: {line: 2, column: 1}- end: {line: 2, column: 10}- range: {start: 10, end: 19}+ - text: "\nSELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 2+ column: 10+ range:+ start: 9+ end: 19+ empty: false+- description: multi-byte characters - Chinese+ input: "SELECT '\u4E2D\u6587';"+ result:+ - text: "SELECT '\u4E2D\u6587';"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 13+ range:+ start: 0+ end: 16+ empty: false+- description: multi-byte characters - emoji+ input: "SELECT '\U0001F389';"+ result:+ - text: "SELECT '\U0001F389';"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 12+ range:+ start: 0+ end: 14empty: false--- description: "multi-byte characters - Chinese"- input: "SELECT '中文';"+- description: multi-byte on second line+ input: "SELECT\n '\u4E2D\u6587';"result:- - text: "SELECT '中文';"+ - text: "SELECT\n '\u4E2D\u6587';"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 13}- range: {start: 0, end: 16}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 8+ range:+ start: 0+ end: 18empty: false--- description: "multi-byte characters - emoji"- input: "SELECT '🎉';"+- description: multiple statements with multi-byte on separate lines+ input: "SELECT '\u4E2D';\nSELECT '\u6587';"result:- - text: "SELECT '🎉';"+ - text: "SELECT '\u4E2D';"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 12}- range: {start: 0, end: 14}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 12+ range:+ start: 0+ end: 13empty: false--- description: "multi-byte on second line"- input: "SELECT\n '中文';"+ - text: "\nSELECT '\u6587';"+ baseline: 0+ start:+ line: 1+ column: 12+ end:+ line: 2+ column: 12+ range:+ start: 13+ end: 27+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"result:- - text: "SELECT\n '中文';"+ - text: "\n\nSELECT 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 8}- range: {start: 0, end: 18}+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11empty: false--- description: "multiple statements with multi-byte on separate lines"- input: "SELECT '中';\nSELECT '文';"+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"result:- - text: "SELECT '中';"+ - text: "\n \n SELECT 1;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 12}- range: {start: 0, end: 13}+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15empty: false- - text: "SELECT '文';"- baseline: 1- start: {line: 2, column: 1}- end: {line: 2, column: 12}- range: {start: 14, end: 27}+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: "SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/tidb/test-data/test_backup.yaml b/backend/plugin/parser/tidb/test-data/test_backup.yamlindex a0af5507c4..7576cbd1ad 100644--- a/backend/plugin/parser/tidb/test-data/test_backup.yaml+++ b/backend/plugin/parser/tidb/test-data/test_backup.yaml@@ -54,8 +54,8 @@sourcetablename: t2targettablename: _rollback_1_t2startposition:- line: 2- column: 1+ line: 1+ column: 39endposition:line: 2column: 39@@ -244,8 +244,8 @@sourcetablename: testtargettablename: _rollback_1_teststartposition:- line: 2- column: 1+ line: 1+ column: 47endposition:line: 2column: 47diff --git a/backend/plugin/parser/tidb/test-data/test_split.yaml b/backend/plugin/parser/tidb/test-data/test_split.yamlindex 18ce18cd61..f7aad5bbba 100644--- a/backend/plugin/parser/tidb/test-data/test_split.yaml+++ b/backend/plugin/parser/tidb/test-data/test_split.yaml@@ -1,91 +1,209 @@-- description: "simple single statement"- input: "SELECT 1;"+- description: simple single statement+ input: SELECT 1;result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false--- description: "multi-line statement"- input: "SELECT\n 1;"+- description: multi-line statement+ input: |-+ SELECT+ 1;result:- - text: "SELECT\n 1;"+ - text: |-+ SELECT+ 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 5}- range: {start: 0, end: 11}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 5+ range:+ start: 0+ end: 11empty: false--- description: "multiple statements"+- description: multiple statementsinput: "SELECT 1;\nSELECT 2;"result:- - text: "SELECT 1;"+ - text: SELECT 1;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 10}- range: {start: 0, end: 9}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9empty: false- - text: "SELECT 2;"- baseline: 1- start: {line: 2, column: 1}- end: {line: 2, column: 10}- range: {start: 10, end: 19}+ - text: "\nSELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 2+ column: 10+ range:+ start: 9+ end: 19empty: false--- description: "multi-byte characters - Chinese"- input: "SELECT '中文';"+- description: multi-byte characters - Chinese+ input: SELECT '中文';result:- - text: "SELECT '中文';"+ - text: SELECT '中文';baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 13}- range: {start: 0, end: 16}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 13+ range:+ start: 0+ end: 16empty: false--- description: "multi-byte characters - emoji"- input: "SELECT '🎉';"+- description: multi-byte characters - emoji+ input: "SELECT '\U0001F389';"result:- - text: "SELECT '🎉';"+ - text: "SELECT '\U0001F389';"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 12}- range: {start: 0, end: 14}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 12+ range:+ start: 0+ end: 14empty: false--- description: "multi-byte on second line"- input: "SELECT\n '中文';"+- description: multi-byte on second line+ input: |-+ SELECT+ '中文';result:- - text: "SELECT\n '中文';"+ - text: |-+ SELECT+ '中文';baseline: 0- start: {line: 1, column: 1}- end: {line: 2, column: 8}- range: {start: 0, end: 18}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 8+ range:+ start: 0+ end: 18empty: false--- description: "multiple statements with multi-byte"- input: "SELECT '中';\nSELECT '文';"+- description: multiple statements with multi-byte+ input: "SELECT '\u4E2D';\nSELECT '\u6587';"result:- - text: "SELECT '中';"+ - text: "SELECT '\u4E2D';"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 12}- range: {start: 0, end: 13}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 12+ range:+ start: 0+ end: 13empty: false- - text: "SELECT '文';"- baseline: 1- start: {line: 2, column: 1}- end: {line: 2, column: 12}- range: {start: 14, end: 27}+ - text: "\nSELECT '\u6587';"+ baseline: 0+ start:+ line: 1+ column: 12+ end:+ line: 2+ column: 12+ range:+ start: 13+ end: 27empty: false--- description: "statement with leading spaces and multi-byte"- input: " SELECT '中';"+- description: statement with leading spaces and multi-byte+ input: ' SELECT ''中'';'result:- - text: "SELECT '中';"+ - text: ' SELECT ''中'';'+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 14+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"baseline: 0- start: {line: 1, column: 3}- end: {line: 1, column: 14}- range: {start: 2, end: 15}+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/tidb/tidb.go b/backend/plugin/parser/tidb/tidb.goindex 450ee80a00..8cb9a2574f 100644--- a/backend/plugin/parser/tidb/tidb.go+++ b/backend/plugin/parser/tidb/tidb.go@@ -57,15 +57,25 @@ func ParseTiDBForSyntaxCheck(statement string) ([]base.AST, error) {}node := nodes[0]- node.SetText(nil, singleSQL.Text)- node.SetOriginTextPosition(int(singleSQL.End.GetLine()))+ // node.Text() includes leading whitespace from singleSQL.Text.+ // This maintains consistency: Statement.Start points to first char of Statement.Text,+ // and AST position matches Statement position.+ // Trim only at display points (e.g., error messages) where needed.++ // Calculate the start line. The native TiDB parser may strip leading newlines+ // from its internal Text(), so we count how many were stripped.+ nativeText := node.Text()+ leadingNewlinesStripped := strings.Count(singleSQL.Text, "\n") - strings.Count(nativeText, "\n")+ actualStartLine := singleSQL.BaseLine + leadingNewlinesStripped + 1++ node.SetOriginTextPosition(actualStartLine)if n, ok := node.(*ast.CreateTableStmt); ok {if err := SetLineForMySQLCreateTableStmt(n); err != nil {- return nil, errors.Wrapf(err, "failed to set line for create table statement at line %d", singleSQL.End.GetLine())+ return nil, errors.Wrapf(err, "failed to set line for create table statement at line %d", actualStartLine)}}results = append(results, &AST{- StartPosition: &storepb.Position{Line: int32(singleSQL.BaseLine) + 1},+ StartPosition: &storepb.Position{Line: int32(actualStartLine)},Node: node,})}@@ -172,7 +182,9 @@ func SetLineForMySQLCreateTableStmt(node *ast.CreateTableStmt) error {if len(node.Cols) == 0 {return nil}- firstLine := node.OriginTextPosition() - strings.Count(node.Text(), "\n")+ // OriginTextPosition() now stores the first line of the statement (1-based),+ // so we can use it directly as firstLine.+ firstLine := node.OriginTextPosition()return tokenizer.NewTokenizer(node.Text()).SetLineForMySQLCreateTableStmt(node, firstLine)}diff --git a/backend/plugin/parser/tidb/tidb_test.go b/backend/plugin/parser/tidb/tidb_test.goindex 64f2348a40..78f39ef5bc 100644--- a/backend/plugin/parser/tidb/tidb_test.go+++ b/backend/plugin/parser/tidb/tidb_test.go@@ -1,7 +1,6 @@package tidbimport (- "strings""testing"tidbparser "github.com/pingcap/tidb/pkg/parser"@@ -12,7 +11,11 @@ import ()type setLineTestData struct {- statement string+ statement string+ // firstLine is the 1-based line number where the CREATE TABLE statement starts.+ // This is used to set OriginTextPosition, which the tokenizer uses to calculate+ // absolute line numbers for columns and constraints.+ firstLine intcolumnLineList []intconstraintLineList []int}@@ -21,11 +24,13 @@ func TestMySQLCreateTableSetLine(t *testing.T) {tests := []setLineTestData{{statement: "CREATE TABLE t as select * from t1",+ firstLine: 1,columnLineList: []int{},constraintLineList: []int{},},{statement: "CREATE TABLE t like t1",+ firstLine: 1,columnLineList: []int{},constraintLineList: []int{},},@@ -43,12 +48,13 @@ func TestMySQLCreateTableSetLine(t *testing.T) {-- it's a comment.FOREIGN KEY (a, b, c) REFERENCES t1(a, b, c)-+)`,+ firstLine: 1,columnLineList: []int{3, 3, 4, 5},constraintLineList: []int{6, 7, 8, 9, 10, 10, 13},},@@ -65,6 +71,7 @@ func TestMySQLCreateTableSetLine(t *testing.T) {"\r\n" +"FOREIGN KEY (a, b, c) REFERENCES t1(a, b, c)" + "\r\n" +")",+ firstLine: 1,columnLineList: []int{3, 3, 4, 5},constraintLineList: []int{6, 7, 8, 8, 10},},@@ -75,6 +82,7 @@ func TestMySQLCreateTableSetLine(t *testing.T) {b int CHECK(b>1), c int UNIQUE)`,+ firstLine: 1,columnLineList: []int{3, 4, 4},constraintLineList: []int{},},@@ -87,6 +95,7 @@ func TestMySQLCreateTableSetLine(t *testing.T) {UNIQUE(name))`,+ firstLine: 1,columnLineList: []int{3, 4},constraintLineList: []int{5, 6},},@@ -102,7 +111,7 @@ func TestMySQLCreateTableSetLine(t *testing.T) {require.True(t, ok)require.Equal(t, len(test.columnLineList), len(node.Cols))require.Equal(t, len(test.constraintLineList), len(node.Constraints))- node.SetOriginTextPosition(strings.Count(test.statement, "\n") + 1)+ node.SetOriginTextPosition(test.firstLine)err = SetLineForMySQLCreateTableStmt(node)require.NoError(t, err)for i, col := range node.Cols {diff --git a/backend/plugin/parser/tokenizer/tokenizer.go b/backend/plugin/parser/tokenizer/tokenizer.goindex a753deac3b..65afb6fb72 100644--- a/backend/plugin/parser/tokenizer/tokenizer.go+++ b/backend/plugin/parser/tokenizer/tokenizer.go@@ -218,11 +218,12 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {var res []base.Statementdelimiter := []rune{';'}- t.skipBlank()t.emptyStatement = true+ // Record position BEFORE skipping whitespace to include leading whitespace in TextstartPos := t.cursorstartLine := t.line // Track the starting line number (1-based)startColumn := t.getColumn() // Track the starting column (1-based)+ t.skipBlank()for {switch {case t.char(0) == eofRune:@@ -289,13 +290,14 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {},})}- t.skipBlank()if err := t.processStreaming(text); err != nil {return nil, err}+ // Record position BEFORE skipping whitespace to include leading whitespace in TextstartPos = t.pos()startLine = t.line // Update startLine for next statementstartColumn = t.getColumn() // Update startColumn for next statement+ t.skipBlank()t.emptyStatement = true// deal with the DELIMITER statement, see https://dev.mysql.com/doc/refman/8.0/en/stored-programs-defining.htmlcase t.equalWordCaseInsensitive(delimiterRuneList):@@ -325,13 +327,14 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {},})}- t.skipBlank()if err := t.processStreaming(text); err != nil {return nil, err}+ // Record position BEFORE skipping whitespace to include leading whitespace in TextstartPos = t.pos()startLine = t.line // Update startLine for next statementstartColumn = t.getColumn() // Update startColumn for next statement+ t.skipBlank()t.emptyStatement = truecase t.char(0) == '/' && t.char(1) == '*':if err := t.scanComment(); err != nil {@@ -384,11 +387,12 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {var res []base.Statement- t.skipBlank()t.emptyStatement = true+ // Record position BEFORE skipping whitespace to include leading whitespace in TextstartPos := t.cursorfirstStatementLine := t.linefirstStatementColumn := t.getColumn()+ t.skipBlank()for {switch {case t.char(0) == '/' && t.char(1) == '*':@@ -433,13 +437,14 @@ func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {},})}- t.skipBlank()if err := t.processStreaming(text); err != nil {return nil, err}+ // Record position BEFORE skipping whitespace to include leading whitespace in TextstartPos = t.pos()firstStatementLine = t.linefirstStatementColumn = t.getColumn()+ t.skipBlank()t.emptyStatement = truecase t.char(0) == eofRune:s := t.getString(startPos, t.pos())diff --git a/backend/plugin/parser/trino/completion.go b/backend/plugin/parser/trino/completion.goindex 8ac4831a10..021ec58b3e 100644--- a/backend/plugin/parser/trino/completion.go+++ b/backend/plugin/parser/trino/completion.go@@ -1145,8 +1145,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.if caretLine == previousSQLEndLine {// The caret is in the same line as the last line of the previous SQL statement.- // We need to adjust the caret offset.- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)+ newCaretOffset = caretOffset - previousSQLEndColumn + 1}break}diff --git a/backend/plugin/parser/trino/split.go b/backend/plugin/parser/trino/split.goindex 182a79d3d8..520d6c2515 100644--- a/backend/plugin/parser/trino/split.go+++ b/backend/plugin/parser/trino/split.go@@ -61,6 +61,7 @@ func splitByParser(statement string) ([]base.Statement, error) {var result []base.Statementtokens := stream.GetAllTokens()+ byteOffsetStart := 0// Walk through all statementsfor _, stmts := range tree.AllStatements() {@@ -81,55 +82,37 @@ func splitByParser(statement string) ([]base.Statement, error) {}// Find the actual start position- startIdx := startToken.GetTokenIndex()endIdx := stopToken.GetTokenIndex()- // Get the first token (any channel) for baseLine calculation- // This is important because when ANTLR reparses fragments, it sees ALL tokens- // including comments on hidden channels, so baseLine must be from the first token- // of any channel for correct error position mapping: originalLine = antlrLine + baseLine- firstToken := tokens[startIdx]-- // Find the first non-hidden token for accurate start position display- var firstDefaultToken antlr.Token- for i := startIdx; i <= endIdx && i < len(tokens); i++ {- if tokens[i].GetChannel() == antlr.TokenDefaultChannel {- firstDefaultToken = tokens[i]- break- }- }-- if firstDefaultToken == nil {- firstDefaultToken = startToken- }-// Check if there's a semicolon after the statement and include itfinalEndIdx := endIdxif endIdx+1 < len(tokens) && tokens[endIdx+1].GetTokenType() == trinoparser.TrinoLexerSEMICOLON_ {finalEndIdx = endIdx + 1}- // Get the text including any trailing semicolon- text := stream.GetTextFromInterval(antlr.NewInterval(startIdx, finalEndIdx))-// Calculate proper end position (1-based exclusive per proto spec)endToken := tokens[finalEndIdx]- // Use actual token positions for Range instead of cumulative offset- // GetStart() returns byte offset of first character, GetStop() returns byte offset of last character- rangeStart := firstDefaultToken.GetStart()+ // Calculate byte range: include leading whitespace from where previous statement endedrangeEnd := endToken.GetStop() + 1 // exclusive end+ // Include leading whitespace in text by getting from original statement+ // This ensures Start.Line - 1 == BaseLine for proper position mapping+ text := statement[byteOffsetStart:rangeEnd]++ // Calculate start position based on byteOffsetStart (including leading whitespace)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffsetStart)+result = append(result, base.Statement{Text: text,- BaseLine: firstToken.GetLine() - 1,+ BaseLine: startLine,Range: &storepb.Range{- Start: int32(rangeStart),+ Start: int32(byteOffsetStart),End: int32(rangeEnd),},Start: &storepb.Position{- Line: int32(firstDefaultToken.GetLine()), // 1-based (ANTLR line is already 1-based)- Column: int32(firstDefaultToken.GetColumn() + 1), // 1-based (ANTLR column is 0-based)+ Line: int32(startLine + 1), // 1-based+ Column: int32(startColumn + 1), // 1-based},End: &storepb.Position{Line: int32(endToken.GetLine()), // 1-based@@ -137,6 +120,8 @@ func splitByParser(statement string) ([]base.Statement, error) {},Empty: false,})++ byteOffsetStart = rangeEnd}return result, nildiff --git a/backend/plugin/parser/trino/test-data/test_split.yaml b/backend/plugin/parser/trino/test-data/test_split.yamlindex f0c2ee8ea9..94742a160f 100644--- a/backend/plugin/parser/trino/test-data/test_split.yaml+++ b/backend/plugin/parser/trino/test-data/test_split.yaml@@ -1,31 +1,111 @@-- description: "multiple statements same line"- input: "SELECT * FROM users; SELECT * FROM orders;"+- description: multiple statements same line+ input: SELECT * FROM users; SELECT * FROM orders;result:- - text: "SELECT * FROM users;"+ - text: SELECT * FROM users;baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 21}- range: {start: 0, end: 20}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 21+ range:+ start: 0+ end: 20empty: false- - text: "SELECT * FROM orders;"+ - text: " SELECT * FROM orders;"baseline: 0- start: {line: 1, column: 22}- end: {line: 1, column: 43}- range: {start: 21, end: 42}+ start:+ line: 1+ column: 21+ end:+ line: 1+ column: 43+ range:+ start: 20+ end: 42empty: false--- description: "unicode in table names"- input: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);"+- description: unicode in table names+ input: "SELECT * FROM \u8868\u540D; INSERT INTO \u8868 VALUES (1);"result:- - text: "SELECT * FROM 表名;"+ - text: "SELECT * FROM \u8868\u540D;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 18}- range: {start: 0, end: 21}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 18+ range:+ start: 0+ end: 21empty: false- - text: "INSERT INTO 表 VALUES (1);"+ - text: " INSERT INTO \u8868 VALUES (1);"baseline: 0- start: {line: 1, column: 19}- end: {line: 1, column: 44}- range: {start: 22, end: 49}+ start:+ line: 1+ column: 18+ end:+ line: 1+ column: 44+ range:+ start: 21+ end: 49+ empty: false+- description: 'position semantic: leading newlines'+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: 'position semantic: leading spaces and newlines'+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: 'position semantic: multi-statement with leading whitespace'+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: SELECT 1;+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/parser/tsql/completion.go b/backend/plugin/parser/tsql/completion.goindex d10de7bfff..5cdc2d5be6 100644--- a/backend/plugin/parser/tsql/completion.go+++ b/backend/plugin/parser/tsql/completion.go@@ -1320,8 +1320,9 @@ func skipHeadingSQLs(statement string, caretLine int, caretOffset int) (string,newCaretLine = caretLine - previousSQLEndLine + 1 // Convert to 1-based.if caretLine == previousSQLEndLine {// The caret is in the same line as the last line of the previous SQL statement.- // We need to adjust the caret offset.- newCaretOffset = caretOffset - previousSQLEndColumn - 1 // Convert to 0-based.+ // End.Column is 1-based exclusive, so (End.Column - 1) gives 0-based start of next statement.+ // newCaretOffset = caretOffset - (previousSQLEndColumn - 1)+ newCaretOffset = caretOffset - previousSQLEndColumn + 1}break}diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.goindex 4432fa11e2..85322239bf 100644--- a/backend/plugin/parser/tsql/split.go+++ b/backend/plugin/parser/tsql/split.go@@ -75,7 +75,8 @@ func splitByParser(statement string) ([]base.Statement, error) {pos := goStmt.GetStop().GetTokenIndex()stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -88,7 +89,10 @@ func splitByParser(statement string) ([]base.Statement, error) {int32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: false,})byteOffset += stmtByteLength@@ -122,7 +126,8 @@ func splitByParser(statement string) ([]base.Statement, error) {pos := goStmt.GetStop().GetTokenIndex()stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -135,7 +140,10 @@ func splitByParser(statement string) ([]base.Statement, error) {int32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: false,})byteOffset += stmtByteLength@@ -153,7 +161,8 @@ func splitByParser(statement string) ([]base.Statement, error) {pos := goStmt.GetStop().GetTokenIndex()stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -166,7 +175,10 @@ func splitByParser(statement string) ([]base.Statement, error) {int32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: false,})byteOffset += stmtByteLength@@ -186,7 +198,8 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenpos := sqlClause.GetStop().GetTokenIndex()stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -199,7 +212,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: false,})byteOffset += stmtByteLength@@ -209,7 +225,8 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenpos := b.Batch_level_statement().GetStop().GetTokenIndex()stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -222,7 +239,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: false,})byteOffset += stmtByteLength@@ -231,7 +251,8 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenpos := b.Execute_body_batch().GetStop().GetTokenIndex()stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -244,7 +265,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: false,})byteOffset += stmtByteLength@@ -253,7 +277,8 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenpos := sqlClause.GetStop().GetTokenIndex()stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])stmtByteLength := len(stmtText)- antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])+ // Calculate start position from byte offset (first character of Text)+ startLine, startColumn := base.CalculateLineAndColumn(statement, byteOffset)result = append(result, base.Statement{Text: stmtText,BaseLine: tokens[start].GetLine() - 1,@@ -266,7 +291,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenint32(tokens[pos].GetColumn()),tokens[pos].GetText(),),- Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),+ Start: &storepb.Position{+ Line: int32(startLine + 1),+ Column: int32(startColumn + 1),+ },Empty: false,})byteOffset += stmtByteLengthdiff --git a/backend/plugin/parser/tsql/test-data/test_split.yaml b/backend/plugin/parser/tsql/test-data/test_split.yamlindex b41757c1b5..8da8a5560e 100644--- a/backend/plugin/parser/tsql/test-data/test_split.yaml+++ b/backend/plugin/parser/tsql/test-data/test_split.yaml@@ -1,71 +1,186 @@-- description: "GO batch separator with multiple batches"+- description: GO batch separator with multiple batchesinput: "-- first statement\ndeclare @temp table(a int)\n-- second statement\ninsert into @temp values(1)\n-- third statement\nselect * from @temp\n-- go statement\ngo\ngo\ngo\ngo\n\t\t\t"result:- text: "-- first statement\ndeclare @temp table(a int)"baseline: 0- start: {line: 2, column: 1}- end: {line: 2, column: 27}- range: {start: 0, end: 45}+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 27+ range:+ start: 0+ end: 45empty: false- text: "\n-- second statement\ninsert into @temp values(1)"baseline: 1- start: {line: 4, column: 1}- end: {line: 4, column: 28}- range: {start: 45, end: 93}+ start:+ line: 2+ column: 27+ end:+ line: 4+ column: 28+ range:+ start: 45+ end: 93empty: false- text: "\n-- third statement\nselect * from @temp\n-- go statement\ngo"baseline: 3- start: {line: 6, column: 1}- end: {line: 8, column: 3}- range: {start: 93, end: 151}+ start:+ line: 4+ column: 28+ end:+ line: 8+ column: 3+ range:+ start: 93+ end: 151empty: false- text: "\ngo"baseline: 7- start: {line: 9, column: 1}- end: {line: 9, column: 3}- range: {start: 151, end: 154}+ start:+ line: 8+ column: 3+ end:+ line: 9+ column: 3+ range:+ start: 151+ end: 154empty: false- text: "\ngo\ngo"baseline: 8- start: {line: 10, column: 1}- end: {line: 11, column: 3}- range: {start: 154, end: 160}+ start:+ line: 9+ column: 3+ end:+ line: 11+ column: 3+ range:+ start: 154+ end: 160empty: false--- description: "multiple UPDATE statements with blank lines"- input: "\n\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";\n\nUPDATE SalesLT.Address SET PostalCode = 0;\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";\n\n\n\n"+- description: multiple UPDATE statements with blank lines+ input: "\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";\n\nUPDATE SalesLT.Address SET PostalCode = 0;\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";\n\n"result:- - text: "\n\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";"+ - text: "\n\n\n\n\n\n\n\nUPDATE SalesLT.Address SET City = \"Shanghai\";"baseline: 0- start: {line: 10, column: 1}- end: {line: 10, column: 46}- range: {start: 0, end: 54}+ start:+ line: 1+ column: 1+ end:+ line: 9+ column: 46+ range:+ start: 0+ end: 53empty: false- text: "\n\nUPDATE SalesLT.Address SET PostalCode = 0;"- baseline: 9- start: {line: 12, column: 1}- end: {line: 12, column: 43}- range: {start: 54, end: 98}+ baseline: 8+ start:+ line: 9+ column: 46+ end:+ line: 11+ column: 43+ range:+ start: 53+ end: 97empty: false- text: "\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";"- baseline: 11- start: {line: 15, column: 1}- end: {line: 15, column: 69}- range: {start: 98, end: 169}+ baseline: 10+ start:+ line: 11+ column: 43+ end:+ line: 14+ column: 69+ range:+ start: 97+ end: 168empty: false--- description: "unicode in table names"+- description: unicode in table namesinput: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);"result:- text: "SELECT * FROM 表名;"baseline: 0- start: {line: 1, column: 1}- end: {line: 1, column: 18}- range: {start: 0, end: 21}+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 18+ range:+ start: 0+ end: 21empty: false- text: " INSERT INTO 表 VALUES (1);"baseline: 0- start: {line: 1, column: 19}- end: {line: 1, column: 44}- range: {start: 21, end: 49}+ start:+ line: 1+ column: 18+ end:+ line: 1+ column: 44+ range:+ start: 21+ end: 49+ empty: false+- description: "position semantic: leading newlines"+ input: "\n\nSELECT 1;"+ result:+ - text: "\n\nSELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 10+ range:+ start: 0+ end: 11+ empty: false+- description: "position semantic: leading spaces and newlines"+ input: "\n \n SELECT 1;"+ result:+ - text: "\n \n SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 12+ range:+ start: 0+ end: 15+ empty: false+- description: "position semantic: multi-statement with leading whitespace"+ input: "SELECT 1;\n\n SELECT 2;"+ result:+ - text: "SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\n\n SELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 3+ column: 12+ range:+ start: 9+ end: 22empty: falsediff --git a/backend/plugin/schema/oracle/generate_migration_testcontainer_test.go b/backend/plugin/schema/oracle/generate_migration_testcontainer_test.goindex 6fe450da91..22bd58ec99 100644--- a/backend/plugin/schema/oracle/generate_migration_testcontainer_test.go+++ b/backend/plugin/schema/oracle/generate_migration_testcontainer_test.go@@ -1874,8 +1874,18 @@ func executeStatements(db *sql.DB, statements string) error {continue}- // Skip pure comment lines- if strings.HasPrefix(stmt, "--") {+ // Skip statements that contain only comments+ // Strip all comment lines and check if there's actual SQL+ lines := strings.Split(stmt, "\n")+ hasSQL := false+ for _, line := range lines {+ trimmed := strings.TrimSpace(line)+ if trimmed != "" && !strings.HasPrefix(trimmed, "--") {+ hasSQL = true+ break+ }+ }+ if !hasSQL {continue}diff --git a/backend/plugin/schema/tidb/walk_through.go b/backend/plugin/schema/tidb/walk_through.goindex 21db90525b..5ed56f8034 100644--- a/backend/plugin/schema/tidb/walk_through.go+++ b/backend/plugin/schema/tidb/walk_through.go@@ -946,7 +946,9 @@ func tidbCreateTable(d *model.DatabaseMetadata, node *tidbast.CreateTableStmt) *}if node.Select != nil {- content := fmt.Sprintf("CREATE TABLE AS statement is used in \"%s\"", node.Text())+ // Trim leading whitespace for display - node.Text() may include leading whitespace+ // to maintain position consistency, but error messages should show clean SQL.+ content := fmt.Sprintf("CREATE TABLE AS statement is used in \"%s\"", strings.TrimSpace(node.Text()))return &storepb.Advice{Status: storepb.Advice_WARNING,Code: code.StatementCreateTableAs.Int32(),