Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: add Statement.Range field for byte offset tracking (#18530)
Danny Xueec45b0ce522
diff --git a/backend/component/sheet/sheet.go b/backend/component/sheet/sheet.goindex 747f2bd279..0ad645d876 100644--- a/backend/component/sheet/sheet.go+++ b/backend/component/sheet/sheet.go@@ -131,10 +131,7 @@ func getSheetCommandsFromByteOffset(engine storepb.Engine, statement string) []*var sheetCommands []*storepb.Rangefor _, s := range statements {- sheetCommands = append(sheetCommands, &storepb.Range{- Start: int32(s.ByteOffsetStart),- End: int32(s.ByteOffsetEnd),- })+ sheetCommands = append(sheetCommands, s.Range)}return sheetCommands}diff --git a/backend/plugin/db/driver.go b/backend/plugin/db/driver.goindex c6bf39ffd2..fcb01db83d 100644--- a/backend/plugin/db/driver.go+++ b/backend/plugin/db/driver.go@@ -285,6 +285,7 @@ func (o *ExecuteOptions) LogSchemaDumpEnd(derr string) {}}+// LogCommandExecute logs the execution of a command.func (o *ExecuteOptions) LogCommandExecute(commandIndexes []int32, commandText string) {if o == nil || o.CreateTaskRunLog == nil {returndiff --git a/backend/plugin/parser/base/lexer.go b/backend/plugin/parser/base/lexer.goindex d37afd0c35..a62cb9ce7a 100644--- a/backend/plugin/parser/base/lexer.go+++ b/backend/plugin/parser/base/lexer.go@@ -6,6 +6,7 @@ import ("github.com/antlr4-go/antlr/v4""github.com/bytebase/bytebase/backend/common"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store")func GetDefaultChannelTokenType(tokens []antlr.Token, base int, offset int) int {@@ -106,6 +107,10 @@ func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, statemesqls = append(sqls, Statement{Text: bufStr.String(),BaseLine: buf[0].GetLine() - 1, // BaseLine is the offset of the first token+ Range: &storepb.Range{+ Start: int32(buf[0].GetStart()),+ End: int32(buf[len(buf)-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(token.GetLine()),Column: int32(token.GetColumn()),diff --git a/backend/plugin/parser/base/statement.go b/backend/plugin/parser/base/statement.goindex ce18ae61bf..5097811dbe 100644--- a/backend/plugin/parser/base/statement.go+++ b/backend/plugin/parser/base/statement.go@@ -20,14 +20,10 @@ type Statement struct {// End is the inclusive end position of the SQL in the original SQL (1-based).End *storepb.Position- // ByteOffsetStart is the start byte position of the sql.+ // Range is the byte offset range of the SQL in the original SQL.// This field may not be present for every engine.- // ByteOffsetStart is intended for sql execution log display. It may not represent the actual sql that is sent to the database.- ByteOffsetStart int- // ByteOffsetEnd is the end byte position of the sql.- // This field may not be present for every engine.- // ByteOffsetEnd is intended for sql execution log display. It may not represent the actual sql that is sent to the database.- ByteOffsetEnd int+ // Range is intended for sql execution log display. It may not represent the actual sql that is sent to the database.+ Range *storepb.Range}// ParsedStatement is the result of parsing SQL (Statement + AST).diff --git a/backend/plugin/parser/bigquery/split.go b/backend/plugin/parser/bigquery/split.goindex c08063d297..b6da131f29 100644--- a/backend/plugin/parser/bigquery/split.go+++ b/backend/plugin/parser/bigquery/split.go@@ -43,6 +43,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {sqls = append(sqls, base.Statement{Text: bufStr.String(),BaseLine: buf[0].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(buf[0].GetStart()),+ End: int32(buf[len(buf)-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(buf[len(buf)-1].GetLine()),Column: int32(buf[len(buf)-1].GetColumn()),diff --git a/backend/plugin/parser/bigquery/split_test.go b/backend/plugin/parser/bigquery/split_test.goindex 8fb5d7875e..262486dd72 100644--- a/backend/plugin/parser/bigquery/split_test.go+++ b/backend/plugin/parser/bigquery/split_test.go@@ -30,6 +30,7 @@ func TestBigQuerySplitMultiSQL(t *testing.T) {Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 8},Empty: false,+ Range: &storepb.Range{Start: 0, End: 8},},},},@@ -44,6 +45,7 @@ func TestBigQuerySplitMultiSQL(t *testing.T) {Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 9},Empty: false,+ Range: &storepb.Range{Start: 0, End: 9},},{Text: "\n SELECT\n 33;",@@ -51,6 +53,7 @@ func TestBigQuerySplitMultiSQL(t *testing.T) {Start: &storepb.Position{Line: 2, Column: 2},End: &storepb.Position{Line: 3, Column: 4},Empty: false,+ Range: &storepb.Range{Start: 9, End: 22},},},},diff --git a/backend/plugin/parser/cassandra/split_test.go b/backend/plugin/parser/cassandra/split_test.goindex a69ac6ad29..e34e64c53a 100644--- a/backend/plugin/parser/cassandra/split_test.go+++ b/backend/plugin/parser/cassandra/split_test.go@@ -17,6 +17,7 @@ func TestSplitSQL(t *testing.T) {{Text: "SELECT * FROM users;",BaseLine: 0,+ Range: &storepb.Range{Start: 0, End: 20},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 20},Empty: false,@@ -24,6 +25,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tSELECT * FROM orders;",BaseLine: 0,+ Range: &storepb.Range{Start: 20, End: 43},Start: &storepb.Position{Line: 2, Column: 2},End: &storepb.Position{Line: 2, Column: 22},Empty: false,@@ -31,6 +33,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tSELECT * FROM products;",BaseLine: 1,+ Range: &storepb.Range{Start: 43, End: 68},Start: &storepb.Position{Line: 3, Column: 2},End: &storepb.Position{Line: 3, Column: 24},Empty: false,diff --git a/backend/plugin/parser/cosmosdb/split.go b/backend/plugin/parser/cosmosdb/split.goindex cb5d070676..62ce0d23d3 100644--- a/backend/plugin/parser/cosmosdb/split.go+++ b/backend/plugin/parser/cosmosdb/split.go@@ -71,6 +71,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {{Text: statement,BaseLine: 0,+ Range: &storepb.Range{+ Start: 0,+ End: int32(len(statement)),+ },Start: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(firstToken.GetLine()),Column: int32(firstToken.GetColumn()),diff --git a/backend/plugin/parser/mysql/split.go b/backend/plugin/parser/mysql/split.goindex ac55de18a7..eebf948c90 100644--- a/backend/plugin/parser/mysql/split.go+++ b/backend/plugin/parser/mysql/split.go@@ -62,6 +62,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[i]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[i].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[i].GetLine()),Column: int32(tokens[i].GetColumn()),@@ -87,6 +91,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([// Use a single semicolon instead of the user defined delimiter.Text: stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";",BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[newStart-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[newStart-1].GetLine()),Column: int32(tokens[newStart-1].GetColumn()),@@ -110,6 +118,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[endPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[endPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[endPos-1].GetLine()),Column: int32(tokens[endPos-1].GetColumn()),@@ -202,6 +214,10 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Comresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),Column: int32(tokens[pos].GetColumn()),@@ -220,6 +236,10 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Comresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[eofPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),Column: int32(tokens[eofPos-1].GetColumn()),@@ -380,6 +400,10 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]bresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),Column: int32(tokens[pos].GetColumn()),@@ -398,6 +422,10 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]bresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[eofPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),Column: int32(tokens[eofPos-1].GetColumn()),diff --git a/backend/plugin/parser/mysql/split_test.go b/backend/plugin/parser/mysql/split_test.goindex 83c7e1309f..67e132b159 100644--- a/backend/plugin/parser/mysql/split_test.go+++ b/backend/plugin/parser/mysql/split_test.go@@ -67,12 +67,14 @@ func TestMySQLSplitMultiSQL(t *testing.T) {BaseLine: 2,Start: &storepb.Position{Line: 3, Column: 4},End: &storepb.Position{Line: 10, Column: 8},+ Range: &storepb.Range{Start: 17, End: 175},},{Text: ` CALL dorepeat(1000);`,BaseLine: 11,Start: &storepb.Position{Line: 12, Column: 4},End: &storepb.Position{Line: 12, Column: 23},+ Range: &storepb.Range{Start: 191, End: 214},},{Text: `@@ -80,6 +82,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {BaseLine: 11,Start: &storepb.Position{Line: 13, Column: 4},End: &storepb.Position{Line: 13, Column: 12},+ Range: &storepb.Range{Start: 214, End: 227},},{Text: "\n\t\t\t",@@ -88,6 +91,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Start: &storepb.Position{Line: 14, Column: 4},End: &storepb.Position{Line: 14, Column: 3},Empty: true,+ Range: &storepb.Range{Start: 227, End: 231},},},},@@ -305,12 +309,14 @@ func TestMySQLSplitMultiSQL(t *testing.T) {FROM t;`,Start: &storepb.Position{Line: 2, Column: 4},End: &storepb.Position{Line: 103, Column: 10},+ Range: &storepb.Range{Start: 0, End: 1080},},{Text: " SELECT * FROM t;",BaseLine: 102,Start: &storepb.Position{Line: 103, Column: 12},End: &storepb.Position{Line: 103, Column: 27},+ Range: &storepb.Range{Start: 1080, End: 1097},},},},@@ -323,11 +329,13 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: `select * from t;`,Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 16},+ Range: &storepb.Range{Start: 0, End: 16},},{Text: `select "\"" where true;`,Start: &storepb.Position{Line: 1, Column: 17},End: &storepb.Position{Line: 1, Column: 39},+ Range: &storepb.Range{Start: 16, End: 39},},},},@@ -343,6 +351,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Start: &storepb.Position{Line: 3, Column: 1},End: &storepb.Position{Line: 2, Column: 23},Empty: true,+ Range: &storepb.Range{Start: 0, End: 39},},},},@@ -356,6 +365,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: `select * from t;`,End: &storepb.Position{Line: 1, Column: 16},Start: &storepb.Position{Line: 1, Column: 1},+ Range: &storepb.Range{Start: 0, End: 16},},{Text: "\n\t\t\t/* sdfasdf */",@@ -363,6 +373,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {End: &storepb.Position{Line: 2, Column: 4},Start: &storepb.Position{Line: 2, Column: 17},Empty: true,+ Range: &storepb.Range{Start: 16, End: 33},},},},@@ -377,18 +388,21 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: `select * from t;`,Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 16},+ Range: &storepb.Range{Start: 0, End: 16},},{Text: "\n\t\t\t/* sdfasdf */;",End: &storepb.Position{Line: 2, Column: 17},Start: &storepb.Position{Line: 2, Column: 17},Empty: true,+ Range: &storepb.Range{Start: 16, End: 34},},{Text: "\n\t\t\tselect * from t;",BaseLine: 1,End: &storepb.Position{Line: 3, Column: 19},Start: &storepb.Position{Line: 3, Column: 4},+ Range: &storepb.Range{Start: 34, End: 54},},},},@@ -427,6 +441,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {END ;`,Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 14, Column: 7},+ Range: &storepb.Range{Start: 0, End: 268},},},},@@ -439,6 +454,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: bigSQL,Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: int32(len(bigSQL))},+ Range: &storepb.Range{Start: 0, End: int32(len(bigSQL))},},},},@@ -451,11 +467,13 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: " CREATE TABLE t(a int);",Start: &storepb.Position{Line: 1, Column: 5},End: &storepb.Position{Line: 1, Column: 26},+ Range: &storepb.Range{Start: 0, End: 26},},{Text: " CREATE TABLE t1(a int)",Start: &storepb.Position{Line: 1, Column: 28},End: &storepb.Position{Line: 1, Column: 49},+ Range: &storepb.Range{Start: 26, End: 49},},},},@@ -469,11 +487,13 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: "CREATE TABLE `tech_Book`(id int, name varchar(255));",Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 52},+ Range: &storepb.Range{Start: 0, End: 52},},{Text: "\nINSERT INTO `tech_Book` VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');",Start: &storepb.Position{Line: 2, Column: 1},End: &storepb.Position{Line: 2, Column: 78},+ Range: &storepb.Range{Start: 52, End: 131},},},},@@ -492,18 +512,21 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));",Start: &storepb.Position{Line: 3, Column: 7},End: &storepb.Position{Line: 3, Column: 76},+ Range: &storepb.Range{Start: 0, End: 110},},{Text: "\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');",Start: &storepb.Position{Line: 5, Column: 7},BaseLine: 2,End: &storepb.Position{Line: 5, Column: 82},+ Range: &storepb.Range{Start: 110, End: 223},},{Text: "\n\t\t\t\t\t\t# this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');",BaseLine: 4,Start: &storepb.Position{Line: 7, Column: 7},End: &storepb.Position{Line: 7, Column: 82},+ Range: &storepb.Range{Start: 223, End: 335},},},},@@ -525,18 +548,21 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;",Start: &storepb.Position{Line: 2, Column: 7},End: &storepb.Position{Line: 7, Column: 7},+ Range: &storepb.Range{Start: 0, End: 181},},{Text: "\n\t\t\t\t\t\tCALL dorepeat(1000);",BaseLine: 6,Start: &storepb.Position{Line: 8, Column: 7},End: &storepb.Position{Line: 8, Column: 26},+ Range: &storepb.Range{Start: 181, End: 208},},{Text: "\n\t\t\t\t\t\tSELECT @x;",BaseLine: 7,Start: &storepb.Position{Line: 9, Column: 7},End: &storepb.Position{Line: 9, Column: 16},+ Range: &storepb.Range{Start: 208, End: 225},},{Text: "\n\t\t\t\t\t\t",@@ -545,6 +571,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {// TODO(zp): Wait, but why the start position is larger than the end position?End: &storepb.Position{Line: 10, Column: 6},Empty: true,+ Range: &storepb.Range{Start: 225, End: 232},},},},@@ -567,18 +594,21 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Text: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\t/* This is a comment */\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;",End: &storepb.Position{Line: 8, Column: 7},Start: &storepb.Position{Line: 2, Column: 7},+ Range: &storepb.Range{Start: 0, End: 211},},{Text: "\n\t\t\t\t\t\tCALL dorepeat(1000);",BaseLine: 7,Start: &storepb.Position{Line: 9, Column: 7},End: &storepb.Position{Line: 9, Column: 26},+ Range: &storepb.Range{Start: 211, End: 238},},{Text: "\n\t\t\t\t\t\tSELECT @x;",BaseLine: 8,Start: &storepb.Position{Line: 10, Column: 7},End: &storepb.Position{Line: 10, Column: 16},+ Range: &storepb.Range{Start: 238, End: 255},},{Text: "\n\t\t\t\t\t\t",@@ -586,6 +616,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {Start: &storepb.Position{Line: 11, Column: 7},End: &storepb.Position{Line: 11, Column: 6},Empty: true,+ Range: &storepb.Range{Start: 255, End: 262},},},},@@ -600,12 +631,14 @@ func TestMySQLSplitMultiSQL(t *testing.T) {BaseLine: 0,Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 2, Column: 8},+ Range: &storepb.Range{Start: 0, End: 24},},{Text: "\r\nCREATE TABLE t1(b int);",BaseLine: 1,Start: &storepb.Position{Line: 3, Column: 1},End: &storepb.Position{Line: 3, Column: 23},+ Range: &storepb.Range{Start: 24, End: 49},},},},diff --git a/backend/plugin/parser/partiql/split_test.go b/backend/plugin/parser/partiql/split_test.goindex e324cc2768..65bce56ce2 100644--- a/backend/plugin/parser/partiql/split_test.go+++ b/backend/plugin/parser/partiql/split_test.go@@ -19,6 +19,7 @@ func TestSplitSQL(t *testing.T) {{Text: "DELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money';",BaseLine: 0,+ Range: &storepb.Range{Start: 0, End: 70},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 70},Empty: false,@@ -26,6 +27,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money'};",BaseLine: 0,+ Range: &storepb.Range{Start: 70, End: 201},Start: &storepb.Position{Line: 2, Column: 2},End: &storepb.Position{Line: 2, Column: 130},Empty: false,@@ -33,6 +35,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money02'};",BaseLine: 1,+ Range: &storepb.Range{Start: 201, End: 334},Start: &storepb.Position{Line: 3, Column: 2},End: &storepb.Position{Line: 3, Column: 132},Empty: false,@@ -40,6 +43,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money03'};",BaseLine: 2,+ Range: &storepb.Range{Start: 334, End: 467},Start: &storepb.Position{Line: 4, Column: 2},End: &storepb.Position{Line: 4, Column: 132},Empty: false,@@ -47,6 +51,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tDELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money02'",BaseLine: 3,+ Range: &storepb.Range{Start: 467, End: 540},Start: &storepb.Position{Line: 5, Column: 2},End: &storepb.Position{Line: 5, Column: 73},Empty: false,diff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.goindex 402887a22b..113a465fda 100644--- a/backend/plugin/parser/pg/split.go+++ b/backend/plugin/parser/pg/split.go@@ -72,6 +72,10 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),Column: int32(tokens[pos].GetColumn()),@@ -90,6 +94,10 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[eofPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),Column: int32(tokens[eofPos-1].GetColumn()),@@ -191,6 +199,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Staresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),Column: int32(tokens[pos].GetColumn()),@@ -209,6 +221,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Staresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[eofPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),Column: int32(tokens[eofPos-1].GetColumn()),diff --git a/backend/plugin/parser/pg/split_test.go b/backend/plugin/parser/pg/split_test.goindex 0583521ce3..86307aee98 100644--- a/backend/plugin/parser/pg/split_test.go+++ b/backend/plugin/parser/pg/split_test.go@@ -44,6 +44,7 @@ func TestPGSplitMultiSQL(t *testing.T) {{Text: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n\t\t\t",BaseLine: 0,+ Range: &storepb.Range{Start: 0, End: 42},Start: &storepb.Position{Line: 3, Column: 4},End: &storepb.Position{Line: 3, Column: 3},Empty: true,@@ -58,11 +59,13 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: `select * from t;`,+ Range: &storepb.Range{Start: 0, End: 16},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 16},},{Text: "\n\t\t\t/* sdfasdf */",+ Range: &storepb.Range{Start: 16, End: 33},End: &storepb.Position{Line: 2, Column: 4},Start: &storepb.Position{Line: 2, Column: 17},Empty: true,@@ -78,11 +81,13 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: `select * from t;`,+ Range: &storepb.Range{Start: 0, End: 16},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 16},},{Text: "\n\t\t\t/* sdfasdf */;",+ Range: &storepb.Range{Start: 16, End: 34},Start: &storepb.Position{Line: 2, Column: 17},End: &storepb.Position{Line: 2, Column: 17},Empty: true,@@ -90,6 +95,7 @@ func TestPGSplitMultiSQL(t *testing.T) {{Text: "\n\t\t\tselect * from t;",BaseLine: 1,+ Range: &storepb.Range{Start: 34, End: 54},Start: &storepb.Position{Line: 3, Column: 4},End: &storepb.Position{Line: 3, Column: 19},},@@ -102,6 +108,7 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: bigSQL,+ Range: &storepb.Range{Start: 0, End: 1048600},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: int32(len(bigSQL))},},@@ -114,11 +121,13 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: " CREATE TABLE t(a int);",+ Range: &storepb.Range{Start: 0, End: 26},Start: &storepb.Position{Line: 1, Column: 5},End: &storepb.Position{Line: 1, Column: 26},},{Text: " CREATE TABLE t1(a int)",+ Range: &storepb.Range{Start: 26, End: 49},Start: &storepb.Position{Line: 1, Column: 28},End: &storepb.Position{Line: 1, Column: 49},},@@ -132,11 +141,13 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: `CREATE TABLE "tech_Book"(id int, name varchar(255));`,+ Range: &storepb.Range{Start: 0, End: 52},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 52},},{Text: "\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');",+ Range: &storepb.Range{Start: 52, End: 135},Start: &storepb.Position{Line: 2, Column: 7},End: &storepb.Position{Line: 2, Column: 82},},@@ -153,12 +164,14 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));",+ Range: &storepb.Range{Start: 0, End: 112},Start: &storepb.Position{Line: 3, Column: 7},End: &storepb.Position{Line: 3, Column: 78},},{Text: "\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');",BaseLine: 2,+ Range: &storepb.Range{Start: 112, End: 225},Start: &storepb.Position{Line: 5, Column: 7},End: &storepb.Position{Line: 5, Column: 82},},@@ -186,12 +199,14 @@ func TestPGSplitMultiSQL(t *testing.T) {-- this is the commentINSERT INTO tbl VALUES ('fasf_bkdjlfa');$$;`,+ Range: &storepb.Range{Start: 0, End: 303},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 8, Column: 9},},{Text: "\n\t\t\t\t\t\tCREATE TABLE t(a int);",BaseLine: 7,+ Range: &storepb.Range{Start: 303, End: 332},Start: &storepb.Position{Line: 9, Column: 7},End: &storepb.Position{Line: 9, Column: 28},},@@ -219,12 +234,14 @@ func TestPGSplitMultiSQL(t *testing.T) {-- this is the commentINSERT INTO tbl VALUES ('fasf_bkdjlfa');$tag_name$;`,+ Range: &storepb.Range{Start: 0, End: 315},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 8, Column: 17},},{Text: "\n\t\t\t\t\t\tCREATE TABLE t(a int);",BaseLine: 7,+ Range: &storepb.Range{Start: 315, End: 344},Start: &storepb.Position{Line: 9, Column: 7},End: &storepb.Position{Line: 9, Column: 28},},@@ -238,12 +255,14 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: "CREATE TABLE t\r\n(a int);",+ Range: &storepb.Range{Start: 0, End: 24},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 2, Column: 8},},{Text: "\r\nCREATE TABLE t1(b int);",BaseLine: 1,+ Range: &storepb.Range{Start: 24, End: 49},Start: &storepb.Position{Line: 3, Column: 1},End: &storepb.Position{Line: 3, Column: 23},},@@ -260,6 +279,7 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: "INSERT INTO \"public\".\"table\"(\"id\",\"content\")\n\t\t\tVALUES\n\t\t\t(12,'table column name () { :xna,sydfn,,kasdfyn;}; /////test string/// 0'),\n\t\t\t(133,'knuandfan public table id'';create table t(a int, b int);set @text=''\\\\\\\\kdaminxkljasdfiebkla.unkonwn''+''abcdef.xyz\\\\''; local xxxyy.abcddd.mysql @text;------- '),\n\t\t\t(1444,'table t xyz abc a''a\\\\\\\\\\\\\\\\''b\"c>?>xxxxxx%}}%%>c<[[?${12344556778990{%}}cake\\\\');",+ Range: &storepb.Range{Start: 0, End: 400},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 5, Column: 92},},@@ -272,6 +292,7 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: "INSERT INTO t VALUES ('klajfas)",+ Range: &storepb.Range{Start: 0, End: 31},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 23},Empty: false,@@ -285,6 +306,7 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: "INSERT INTO \"t VALUES ('klajfas)",+ Range: &storepb.Range{Start: 0, End: 32},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 13},},@@ -298,6 +320,7 @@ func TestPGSplitMultiSQL(t *testing.T) {{Text: "/*INSERT INTO \"t VALUES ('klajfas)",BaseLine: 0,+ Range: &storepb.Range{Start: 0, End: 34},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 1},Empty: false,@@ -311,6 +334,7 @@ func TestPGSplitMultiSQL(t *testing.T) {res: []base.Statement{{Text: "$$INSERT INTO \"t VALUES ('klajfas)",+ Range: &storepb.Range{Start: 0, End: 34},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 3},},diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.goindex ec00684061..edfc1a600c 100644--- a/backend/plugin/parser/plsql/split.go+++ b/backend/plugin/parser/plsql/split.go@@ -85,9 +85,11 @@ func SplitSQL(statement string) ([]base.Statement, error) {},statement,),- Empty: base.IsEmpty(tokens.GetAllTokens()[stmt.GetStart().GetTokenIndex():stmt.GetStop().GetTokenIndex()+1], parser.PlSqlParserSEMICOLON),- ByteOffsetStart: byteOffsetStart,- ByteOffsetEnd: byteOffsetEnd,+ Empty: base.IsEmpty(tokens.GetAllTokens()[stmt.GetStart().GetTokenIndex():stmt.GetStop().GetTokenIndex()+1], parser.PlSqlParserSEMICOLON),+ Range: &storepb.Range{+ Start: int32(byteOffsetStart),+ End: int32(byteOffsetEnd),+ },})byteOffsetStart = byteOffsetEndprevStopTokenIndex = stmt.GetStop().GetTokenIndex()diff --git a/backend/plugin/parser/plsql/split_test.go b/backend/plugin/parser/plsql/split_test.goindex 973e2e1d22..c79823801d 100644--- a/backend/plugin/parser/plsql/split_test.go+++ b/backend/plugin/parser/plsql/split_test.go@@ -32,20 +32,20 @@ func TestOracleSplitMultiSQL(t *testing.T) {want: resData{res: []base.Statement{{- Text: `select * from t`,- BaseLine: 1,- ByteOffsetStart: 5,- ByteOffsetEnd: 20,- Start: &storepb.Position{Line: 2, Column: 5},- End: &storepb.Position{Line: 2, Column: 19},+ Text: `select * from t`,+ BaseLine: 1,+ Range: &storepb.Range{Start: int32(5),+ End: int32(20)},+ Start: &storepb.Position{Line: 2, Column: 5},+ End: &storepb.Position{Line: 2, Column: 19},},{- Text: `create table table$1 (id int)`,- BaseLine: 2,- ByteOffsetStart: 26,- ByteOffsetEnd: 56,- Start: &storepb.Position{Line: 3, Column: 5},- End: &storepb.Position{Line: 3, Column: 33},+ Text: `create table table$1 (id int)`,+ BaseLine: 2,+ Range: &storepb.Range{Start: int32(26),+ End: int32(56)},+ Start: &storepb.Position{Line: 3, Column: 5},+ End: &storepb.Position{Line: 3, Column: 33},},},},@@ -63,10 +63,10 @@ ONLINE;`,want: resData{res: []base.Statement{{- Text: "ALTER TABLE DATA.TEST\nMODIFY PARTITION BY RANGE (TXN_DATE)\nINTERVAL (NUMTODSINTERVAL(1, 'DAY'))\n(\n\tPARTITION TEST_PO VALUES LESS THAN (\n\t\tTO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')\n\t)\n)\nONLINE",- Start: &storepb.Position{Line: 1, Column: 1},- End: &storepb.Position{Line: 9, Column: 1},- ByteOffsetEnd: 233,+ Text: "ALTER TABLE DATA.TEST\nMODIFY PARTITION BY RANGE (TXN_DATE)\nINTERVAL (NUMTODSINTERVAL(1, 'DAY'))\n(\n\tPARTITION TEST_PO VALUES LESS THAN (\n\t\tTO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')\n\t)\n)\nONLINE",+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 9, Column: 1},+ Range: &storepb.Range{Start: 0, End: 233},},},},@@ -90,10 +90,10 @@ ONLINE;`,want: resData{res: []base.Statement{{- Text: "DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS",- Start: &storepb.Position{Line: 1, Column: 1},- End: &storepb.Position{Line: 1, Column: 48},- ByteOffsetEnd: 58,+ Text: "DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS",+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 48},+ Range: &storepb.Range{Start: 0, End: 58},},},},@@ -103,10 +103,10 @@ ONLINE;`,want: resData{res: []base.Statement{{- Text: "CALL my_procedure()",- Start: &storepb.Position{Line: 1, Column: 1},- End: &storepb.Position{Line: 1, Column: 19},- ByteOffsetEnd: 19,+ Text: "CALL my_procedure()",+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 19},+ Range: &storepb.Range{Start: 0, End: 19},},},},@@ -116,18 +116,18 @@ ONLINE;`,want: resData{res: []base.Statement{{- Text: "SELECT * FROM t1",- Start: &storepb.Position{Line: 1, Column: 1},- End: &storepb.Position{Line: 1, Column: 15},- ByteOffsetStart: 0,- ByteOffsetEnd: 16,+ Text: "SELECT * FROM t1",+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 15},+ Range: &storepb.Range{Start: int32(0),+ End: int32(16)},},{- Text: "SELECT * FROM t2",- Start: &storepb.Position{Line: 1, Column: 19},- End: &storepb.Position{Line: 1, Column: 33},- ByteOffsetStart: 18,- ByteOffsetEnd: 34,+ Text: "SELECT * FROM t2",+ Start: &storepb.Position{Line: 1, Column: 19},+ End: &storepb.Position{Line: 1, Column: 33},+ Range: &storepb.Range{Start: int32(18),+ End: int32(34)},},},},@@ -142,10 +142,10 @@ END;want: resData{res: []base.Statement{{- Text: "CREATE OR REPLACE PROCEDURE test_proc IS\nBEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Hello');\nEND;",- Start: &storepb.Position{Line: 1, Column: 1},- End: &storepb.Position{Line: 4, Column: 4},- ByteOffsetEnd: 91,+ Text: "CREATE OR REPLACE PROCEDURE test_proc IS\nBEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Hello');\nEND;",+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 4, Column: 4},+ Range: &storepb.Range{Start: 0, End: 91},},},},@@ -164,20 +164,20 @@ END;want: resData{res: []base.Statement{{- Text: "CREATE OR REPLACE PROCEDURE proc1 IS\nBEGIN\n NULL;\nEND;",- BaseLine: 0,- Start: &storepb.Position{Line: 1, Column: 1},- End: &storepb.Position{Line: 4, Column: 4},- ByteOffsetStart: 0,- ByteOffsetEnd: 57,+ Text: "CREATE OR REPLACE PROCEDURE proc1 IS\nBEGIN\n NULL;\nEND;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 4, Column: 4},+ Range: &storepb.Range{Start: int32(0),+ End: int32(57)},},{- Text: "CREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;",- BaseLine: 5,- Start: &storepb.Position{Line: 6, Column: 1},- End: &storepb.Position{Line: 9, Column: 4},- ByteOffsetStart: 60,- ByteOffsetEnd: 117,+ Text: "CREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;",+ BaseLine: 5,+ Start: &storepb.Position{Line: 6, Column: 1},+ End: &storepb.Position{Line: 9, Column: 4},+ Range: &storepb.Range{Start: int32(60),+ End: int32(117)},},},},@@ -189,20 +189,20 @@ SELECT * FROM t2;`,want: resData{res: []base.Statement{{- Text: "SELECT * FROM t1",- BaseLine: 0,- Start: &storepb.Position{Line: 1, Column: 1},- End: &storepb.Position{Line: 1, Column: 15},- ByteOffsetStart: 0,- ByteOffsetEnd: 16,+ Text: "SELECT * FROM t1",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 15},+ Range: &storepb.Range{Start: int32(0),+ End: int32(16)},},{- Text: "SELECT * FROM t2",- BaseLine: 2,- Start: &storepb.Position{Line: 3, Column: 1},- End: &storepb.Position{Line: 3, Column: 15},- ByteOffsetStart: 20,- ByteOffsetEnd: 36,+ Text: "SELECT * FROM t2",+ BaseLine: 2,+ Start: &storepb.Position{Line: 3, Column: 1},+ End: &storepb.Position{Line: 3, Column: 15},+ Range: &storepb.Range{Start: int32(20),+ End: int32(36)},},},},@@ -215,10 +215,10 @@ END;want: resData{res: []base.Statement{{- Text: "BEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Test');\nEND;",- Start: &storepb.Position{Line: 1, Column: 1},- End: &storepb.Position{Line: 3, Column: 4},- ByteOffsetEnd: 49,+ Text: "BEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Test');\nEND;",+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 3, Column: 4},+ Range: &storepb.Range{Start: 0, End: 49},},},},diff --git a/backend/plugin/parser/redshift/split.go b/backend/plugin/parser/redshift/split.goindex 1373c1c7d3..521d6e7043 100644--- a/backend/plugin/parser/redshift/split.go+++ b/backend/plugin/parser/redshift/split.go@@ -73,6 +73,10 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),Column: int32(tokens[pos].GetColumn()),@@ -91,6 +95,10 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[eofPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),Column: int32(tokens[eofPos-1].GetColumn()),@@ -191,6 +199,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Staresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),Column: int32(tokens[pos].GetColumn()),@@ -209,6 +221,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Staresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[eofPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),Column: int32(tokens[eofPos-1].GetColumn()),diff --git a/backend/plugin/parser/snowflake/split_test.go b/backend/plugin/parser/snowflake/split_test.goindex a12282729f..7a87ab2cd6 100644--- a/backend/plugin/parser/snowflake/split_test.go+++ b/backend/plugin/parser/snowflake/split_test.go@@ -17,6 +17,7 @@ func TestSplitSQL(t *testing.T) {{Text: "SELECT * FROM users;",BaseLine: 0,+ Range: &storepb.Range{Start: 0, End: 20},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 20},Empty: false,@@ -24,6 +25,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tSELECT * FROM orders;",BaseLine: 0,+ Range: &storepb.Range{Start: 20, End: 43},Start: &storepb.Position{Line: 2, Column: 2},End: &storepb.Position{Line: 2, Column: 22},Empty: false,@@ -31,6 +33,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tSELECT * FROM products;",BaseLine: 1,+ Range: &storepb.Range{Start: 43, End: 68},Start: &storepb.Position{Line: 3, Column: 2},End: &storepb.Position{Line: 3, Column: 24},Empty: false,diff --git a/backend/plugin/parser/spanner/split.go b/backend/plugin/parser/spanner/split.goindex b6f586b419..4068b0964f 100644--- a/backend/plugin/parser/spanner/split.go+++ b/backend/plugin/parser/spanner/split.go@@ -92,6 +92,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[endPos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[endPos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[endPos].GetLine()),Column: int32(tokens[endPos].GetColumn()),diff --git a/backend/plugin/parser/spanner/split_test.go b/backend/plugin/parser/spanner/split_test.goindex 260ba442fb..02f738dc1e 100644--- a/backend/plugin/parser/spanner/split_test.go+++ b/backend/plugin/parser/spanner/split_test.go@@ -17,6 +17,7 @@ func TestSplitSQL(t *testing.T) {{Text: "SELECT * FROM users;",BaseLine: 0,+ Range: &storepb.Range{Start: 0, End: 20},Start: &storepb.Position{Line: 1, Column: 1},End: &storepb.Position{Line: 1, Column: 20},Empty: false,@@ -24,6 +25,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tSELECT * FROM orders;",BaseLine: 0,+ Range: &storepb.Range{Start: 20, End: 43},Start: &storepb.Position{Line: 2, Column: 2},End: &storepb.Position{Line: 2, Column: 22},Empty: false,@@ -31,6 +33,7 @@ func TestSplitSQL(t *testing.T) {{Text: "\n\tSELECT * FROM products;",BaseLine: 1,+ Range: &storepb.Range{Start: 43, End: 68},Start: &storepb.Position{Line: 3, Column: 2},End: &storepb.Position{Line: 3, Column: 24},Empty: false,diff --git a/backend/plugin/parser/standard/split.go b/backend/plugin/parser/standard/split.goindex 756f9cd57d..08b308d6f6 100644--- a/backend/plugin/parser/standard/split.go+++ b/backend/plugin/parser/standard/split.go@@ -23,12 +23,27 @@ func init() {// SplitSQL splits the given SQL statement into multiple SQL statements.func SplitSQL(statement string) ([]base.Statement, error) {var list []base.Statement+ byteOffset := 0err := applyMultiStatements(strings.NewReader(statement), func(sql string) error {+ // Find the start position of this SQL in the original statement+ startPos := strings.Index(statement[byteOffset:], strings.TrimLeft(sql, "\n\t "))+ if startPos != -1 {+ startPos += byteOffset+ } else {+ startPos = byteOffset+ }+ endPos := startPos + len(sql)+list = append(list, base.Statement{- Text: sql,+ Text: sql,+ Range: &storepb.Range{+ Start: int32(startPos),+ End: int32(endPos),+ },End: &storepb.Position{Line: 0, Column: 0},Empty: false,})+ byteOffset = endPosreturn nil})return list, errdiff --git a/backend/plugin/parser/tidb/split.go b/backend/plugin/parser/tidb/split.goindex abf0164d24..a940a93a5f 100644--- a/backend/plugin/parser/tidb/split.go+++ b/backend/plugin/parser/tidb/split.go@@ -97,6 +97,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[i]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[i].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[i].GetLine()),@@ -125,6 +129,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([// Use a single semicolon instead of the user defined delimiter.Text: stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";",BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[newStart-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[newStart-1].GetLine()),@@ -151,6 +159,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[endPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[endPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[endPos-1].GetLine()),@@ -315,6 +327,10 @@ func splitTiDBStatement(stream *antlr.CommonTokenStream, statement string) ([]baresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -336,6 +352,10 @@ func splitTiDBStatement(stream *antlr.CommonTokenStream, statement string) ([]baresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[eofPos-1].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),diff --git a/backend/plugin/parser/tokenizer/tokenizer.go b/backend/plugin/parser/tokenizer/tokenizer.goindex 146d02ee8e..98477a1fbc 100644--- a/backend/plugin/parser/tokenizer/tokenizer.go+++ b/backend/plugin/parser/tokenizer/tokenizer.go@@ -246,10 +246,12 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {// but we want to get the line of last line of the SQL// which means the line of ')'.// So we need minus the aboveNonBlankLineDistance.- End: &store.Position{Line: int32(t.line - t.aboveNonBlankLineDistance())},- Empty: t.emptyStatement,- ByteOffsetStart: t.getByteOffset(int(startPos)),- ByteOffsetEnd: t.getByteOffset(int(t.pos())),+ End: &store.Position{Line: int32(t.line - t.aboveNonBlankLineDistance())},+ Empty: t.emptyStatement,+ Range: &store.Range{+ Start: int32(t.getByteOffset(int(startPos))),+ End: int32(t.getByteOffset(int(t.pos()))),+ },})}if err := t.processStreaming(s); err != nil {@@ -265,11 +267,13 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {res = append(res, base.Statement{Text: text,// BaseLine is 0-based line number, so subtract 1 from 1-based startLine- BaseLine: startLine - 1,- End: &store.Position{Line: int32(t.line)},- Empty: t.emptyStatement,- ByteOffsetStart: t.getByteOffset(int(startPos)),- ByteOffsetEnd: t.getByteOffset(int(t.pos())),+ BaseLine: startLine - 1,+ End: &store.Position{Line: int32(t.line)},+ Empty: t.emptyStatement,+ Range: &store.Range{+ Start: int32(t.getByteOffset(int(startPos))),+ End: int32(t.getByteOffset(int(t.pos()))),+ },})}t.skipBlank()@@ -291,11 +295,13 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {res = append(res, base.Statement{Text: text,// BaseLine is 0-based line number, so subtract 1 from 1-based startLine- BaseLine: startLine - 1,- End: &store.Position{Line: int32(t.line)},- Empty: false,- ByteOffsetStart: t.getByteOffset(int(startPos)),- ByteOffsetEnd: t.getByteOffset(int(t.pos())),+ BaseLine: startLine - 1,+ End: &store.Position{Line: int32(t.line)},+ Empty: false,+ Range: &store.Range{+ Start: int32(t.getByteOffset(int(startPos))),+ End: int32(t.getByteOffset(int(t.pos()))),+ },})}t.skipBlank()@@ -398,6 +404,10 @@ func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {Column: int32(firstStatementColumn),},Empty: t.emptyStatement,+ Range: &store.Range{+ Start: int32(t.getByteOffset(int(startPos))),+ End: int32(t.getByteOffset(int(t.pos()))),+ },})}t.skipBlank()@@ -433,6 +443,10 @@ func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {Column: int32(firstStatementColumn),},Empty: t.emptyStatement,+ Range: &store.Range{+ Start: int32(t.getByteOffset(int(startPos))),+ End: int32(t.getByteOffset(int(t.pos()))),+ },})}if err := t.processStreaming(s); err != nil {@@ -510,6 +524,10 @@ func (t *Tokenizer) SplitPostgreSQLMultiSQL() ([]base.Statement, error) {BaseLine: startLine - 1,End: &store.Position{Line: int32(t.line)},Empty: t.emptyStatement,+ Range: &store.Range{+ Start: int32(t.getByteOffset(int(startPos))),+ End: int32(t.getByteOffset(int(t.pos()))),+ },})}t.skipBlank()@@ -543,6 +561,10 @@ func (t *Tokenizer) SplitPostgreSQLMultiSQL() ([]base.Statement, error) {Line: int32(t.line - t.aboveNonBlankLineDistance()),},Empty: t.emptyStatement,+ Range: &store.Range{+ Start: int32(t.getByteOffset(int(startPos))),+ End: int32(t.getByteOffset(int(t.pos()))),+ },})}if err := t.processStreaming(s); err != nil {diff --git a/backend/plugin/parser/trino/split.go b/backend/plugin/parser/trino/split.goindex ab5d47acb9..3727ef128a 100644--- a/backend/plugin/parser/trino/split.go+++ b/backend/plugin/parser/trino/split.go@@ -191,6 +191,10 @@ func splitByParser(statement string) ([]base.Statement, error) {result = append(result, base.Statement{Text: text,BaseLine: firstToken.GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[startIdx].GetStart()),+ End: int32(endToken.GetStop() + 1),+ },Start: &storepb.Position{Line: int32(firstDefaultToken.GetLine() - 1),Column: int32(firstDefaultToken.GetColumn()),diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.goindex c2f0e5cd04..d8f8daf6c4 100644--- a/backend/plugin/parser/tsql/split.go+++ b/backend/plugin/parser/tsql/split.go@@ -76,6 +76,10 @@ func splitByParser(statement string) ([]base.Statement, error) {result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -118,6 +122,10 @@ func splitByParser(statement string) ([]base.Statement, error) {result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -144,6 +152,10 @@ func splitByParser(statement string) ([]base.Statement, error) {result = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -172,6 +184,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -190,6 +206,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -207,6 +227,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -224,6 +248,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Tokenresult = append(result, base.Statement{Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),BaseLine: tokens[start].GetLine() - 1,+ Range: &storepb.Range{+ Start: int32(tokens[start].GetStart()),+ End: int32(tokens[pos].GetStop() + 1),+ },End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),diff --git a/backend/plugin/parser/tsql/split_test.go b/backend/plugin/parser/tsql/split_test.goindex 9a4028ac14..38b218d925 100644--- a/backend/plugin/parser/tsql/split_test.go+++ b/backend/plugin/parser/tsql/split_test.go@@ -43,6 +43,7 @@ goStart: &storepb.Position{Line: 2, Column: 1},End: &storepb.Position{Line: 2, Column: 26},Empty: false,+ Range: &storepb.Range{Start: 0, End: 45},},{Text: "\n-- second statement\ninsert into @temp values(1)",@@ -50,6 +51,7 @@ goStart: &storepb.Position{Line: 4, Column: 1},End: &storepb.Position{Line: 4, Column: 27},Empty: false,+ Range: &storepb.Range{Start: 45, End: 93},},{Text: "\n-- third statement\nselect * from @temp\n-- go statement\ngo",@@ -57,6 +59,7 @@ goStart: &storepb.Position{Line: 6, Column: 1},End: &storepb.Position{Line: 8, Column: 1},Empty: false,+ Range: &storepb.Range{Start: 93, End: 151},},{Text: "\ngo",@@ -64,6 +67,7 @@ goStart: &storepb.Position{Line: 9, Column: 1},End: &storepb.Position{Line: 9, Column: 1},Empty: false,+ Range: &storepb.Range{Start: 151, End: 154},},{Text: "\ngo\ngo",@@ -71,6 +75,7 @@ goStart: &storepb.Position{Line: 10, Column: 1},End: &storepb.Position{Line: 11, Column: 1},Empty: false,+ Range: &storepb.Range{Start: 154, End: 160},},},},@@ -103,6 +108,7 @@ UPDATE SalesLT.ProductModelProductDescription SET Culture = "zh-cn";Start: &storepb.Position{Line: 10, Column: 1},End: &storepb.Position{Line: 10, Column: 45},Empty: false,+ Range: &storepb.Range{Start: 0, End: 54},},{Text: "\n\nUPDATE SalesLT.Address SET PostalCode = 0;",@@ -110,6 +116,7 @@ UPDATE SalesLT.ProductModelProductDescription SET Culture = "zh-cn";Start: &storepb.Position{Line: 12, Column: 1},End: &storepb.Position{Line: 12, Column: 42},Empty: false,+ Range: &storepb.Range{Start: 54, End: 98},},{Text: "\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";",@@ -117,6 +124,7 @@ UPDATE SalesLT.ProductModelProductDescription SET Culture = "zh-cn";Start: &storepb.Position{Line: 15, Column: 1},End: &storepb.Position{Line: 15, Column: 68},Empty: false,+ Range: &storepb.Range{Start: 98, End: 169},},},},