Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: add Statement.Range field for byte offset tracking (#18530)

Danny Xueec45b0ce522
diff --git a/backend/component/sheet/sheet.go b/backend/component/sheet/sheet.go
index 747f2bd279..0ad645d876 100644
--- a/backend/component/sheet/sheet.go
+++ b/backend/component/sheet/sheet.go
@@ -131,10 +131,7 @@ func getSheetCommandsFromByteOffset(engine storepb.Engine, statement string) []*
var sheetCommands []*storepb.Range
for _, s := range statements {
- sheetCommands = append(sheetCommands, &storepb.Range{
- Start: int32(s.ByteOffsetStart),
- End: int32(s.ByteOffsetEnd),
- })
+ sheetCommands = append(sheetCommands, s.Range)
}
return sheetCommands
}
diff --git a/backend/plugin/db/driver.go b/backend/plugin/db/driver.go
index c6bf39ffd2..fcb01db83d 100644
--- a/backend/plugin/db/driver.go
+++ b/backend/plugin/db/driver.go
@@ -285,6 +285,7 @@ func (o *ExecuteOptions) LogSchemaDumpEnd(derr string) {
}
}
+// LogCommandExecute logs the execution of a command.
func (o *ExecuteOptions) LogCommandExecute(commandIndexes []int32, commandText string) {
if o == nil || o.CreateTaskRunLog == nil {
return
diff --git a/backend/plugin/parser/base/lexer.go b/backend/plugin/parser/base/lexer.go
index d37afd0c35..a62cb9ce7a 100644
--- a/backend/plugin/parser/base/lexer.go
+++ b/backend/plugin/parser/base/lexer.go
@@ -6,6 +6,7 @@ import (
"github.com/antlr4-go/antlr/v4"
"github.com/bytebase/bytebase/backend/common"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
func GetDefaultChannelTokenType(tokens []antlr.Token, base int, offset int) int {
@@ -106,6 +107,10 @@ func SplitSQLByLexer(stream *antlr.CommonTokenStream, semiTokenType int, stateme
sqls = append(sqls, Statement{
Text: bufStr.String(),
BaseLine: buf[0].GetLine() - 1, // BaseLine is the offset of the first token
+ Range: &storepb.Range{
+ Start: int32(buf[0].GetStart()),
+ End: int32(buf[len(buf)-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(token.GetLine()),
Column: int32(token.GetColumn()),
diff --git a/backend/plugin/parser/base/statement.go b/backend/plugin/parser/base/statement.go
index ce18ae61bf..5097811dbe 100644
--- a/backend/plugin/parser/base/statement.go
+++ b/backend/plugin/parser/base/statement.go
@@ -20,14 +20,10 @@ type Statement struct {
// End is the inclusive end position of the SQL in the original SQL (1-based).
End *storepb.Position
- // ByteOffsetStart is the start byte position of the sql.
+ // Range is the byte offset range of the SQL in the original SQL.
// This field may not be present for every engine.
- // ByteOffsetStart is intended for sql execution log display. It may not represent the actual sql that is sent to the database.
- ByteOffsetStart int
- // ByteOffsetEnd is the end byte position of the sql.
- // This field may not be present for every engine.
- // ByteOffsetEnd is intended for sql execution log display. It may not represent the actual sql that is sent to the database.
- ByteOffsetEnd int
+ // Range is intended for sql execution log display. It may not represent the actual sql that is sent to the database.
+ Range *storepb.Range
}
// ParsedStatement is the result of parsing SQL (Statement + AST).
diff --git a/backend/plugin/parser/bigquery/split.go b/backend/plugin/parser/bigquery/split.go
index c08063d297..b6da131f29 100644
--- a/backend/plugin/parser/bigquery/split.go
+++ b/backend/plugin/parser/bigquery/split.go
@@ -43,6 +43,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {
sqls = append(sqls, base.Statement{
Text: bufStr.String(),
BaseLine: buf[0].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(buf[0].GetStart()),
+ End: int32(buf[len(buf)-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(buf[len(buf)-1].GetLine()),
Column: int32(buf[len(buf)-1].GetColumn()),
diff --git a/backend/plugin/parser/bigquery/split_test.go b/backend/plugin/parser/bigquery/split_test.go
index 8fb5d7875e..262486dd72 100644
--- a/backend/plugin/parser/bigquery/split_test.go
+++ b/backend/plugin/parser/bigquery/split_test.go
@@ -30,6 +30,7 @@ func TestBigQuerySplitMultiSQL(t *testing.T) {
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 8},
Empty: false,
+ Range: &storepb.Range{Start: 0, End: 8},
},
},
},
@@ -44,6 +45,7 @@ func TestBigQuerySplitMultiSQL(t *testing.T) {
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 9},
Empty: false,
+ Range: &storepb.Range{Start: 0, End: 9},
},
{
Text: "\n SELECT\n 33;",
@@ -51,6 +53,7 @@ func TestBigQuerySplitMultiSQL(t *testing.T) {
Start: &storepb.Position{Line: 2, Column: 2},
End: &storepb.Position{Line: 3, Column: 4},
Empty: false,
+ Range: &storepb.Range{Start: 9, End: 22},
},
},
},
diff --git a/backend/plugin/parser/cassandra/split_test.go b/backend/plugin/parser/cassandra/split_test.go
index a69ac6ad29..e34e64c53a 100644
--- a/backend/plugin/parser/cassandra/split_test.go
+++ b/backend/plugin/parser/cassandra/split_test.go
@@ -17,6 +17,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "SELECT * FROM users;",
BaseLine: 0,
+ Range: &storepb.Range{Start: 0, End: 20},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 20},
Empty: false,
@@ -24,6 +25,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tSELECT * FROM orders;",
BaseLine: 0,
+ Range: &storepb.Range{Start: 20, End: 43},
Start: &storepb.Position{Line: 2, Column: 2},
End: &storepb.Position{Line: 2, Column: 22},
Empty: false,
@@ -31,6 +33,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tSELECT * FROM products;",
BaseLine: 1,
+ Range: &storepb.Range{Start: 43, End: 68},
Start: &storepb.Position{Line: 3, Column: 2},
End: &storepb.Position{Line: 3, Column: 24},
Empty: false,
diff --git a/backend/plugin/parser/cosmosdb/split.go b/backend/plugin/parser/cosmosdb/split.go
index cb5d070676..62ce0d23d3 100644
--- a/backend/plugin/parser/cosmosdb/split.go
+++ b/backend/plugin/parser/cosmosdb/split.go
@@ -71,6 +71,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {
{
Text: statement,
BaseLine: 0,
+ Range: &storepb.Range{
+ Start: 0,
+ End: int32(len(statement)),
+ },
Start: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(firstToken.GetLine()),
Column: int32(firstToken.GetColumn()),
diff --git a/backend/plugin/parser/mysql/split.go b/backend/plugin/parser/mysql/split.go
index ac55de18a7..eebf948c90 100644
--- a/backend/plugin/parser/mysql/split.go
+++ b/backend/plugin/parser/mysql/split.go
@@ -62,6 +62,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[i]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[i].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[i].GetLine()),
Column: int32(tokens[i].GetColumn()),
@@ -87,6 +91,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
// Use a single semicolon instead of the user defined delimiter.
Text: stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";",
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[newStart-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[newStart-1].GetLine()),
Column: int32(tokens[newStart-1].GetColumn()),
@@ -110,6 +118,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[endPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[endPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[endPos-1].GetLine()),
Column: int32(tokens[endPos-1].GetColumn()),
@@ -202,6 +214,10 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Com
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
Column: int32(tokens[pos].GetColumn()),
@@ -220,6 +236,10 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Com
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[eofPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
Column: int32(tokens[eofPos-1].GetColumn()),
@@ -380,6 +400,10 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
Column: int32(tokens[pos].GetColumn()),
@@ -398,6 +422,10 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[eofPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
Column: int32(tokens[eofPos-1].GetColumn()),
diff --git a/backend/plugin/parser/mysql/split_test.go b/backend/plugin/parser/mysql/split_test.go
index 83c7e1309f..67e132b159 100644
--- a/backend/plugin/parser/mysql/split_test.go
+++ b/backend/plugin/parser/mysql/split_test.go
@@ -67,12 +67,14 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
BaseLine: 2,
Start: &storepb.Position{Line: 3, Column: 4},
End: &storepb.Position{Line: 10, Column: 8},
+ Range: &storepb.Range{Start: 17, End: 175},
},
{
Text: ` CALL dorepeat(1000);`,
BaseLine: 11,
Start: &storepb.Position{Line: 12, Column: 4},
End: &storepb.Position{Line: 12, Column: 23},
+ Range: &storepb.Range{Start: 191, End: 214},
},
{
Text: `
@@ -80,6 +82,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
BaseLine: 11,
Start: &storepb.Position{Line: 13, Column: 4},
End: &storepb.Position{Line: 13, Column: 12},
+ Range: &storepb.Range{Start: 214, End: 227},
},
{
Text: "\n\t\t\t",
@@ -88,6 +91,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Start: &storepb.Position{Line: 14, Column: 4},
End: &storepb.Position{Line: 14, Column: 3},
Empty: true,
+ Range: &storepb.Range{Start: 227, End: 231},
},
},
},
@@ -305,12 +309,14 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
FROM t;`,
Start: &storepb.Position{Line: 2, Column: 4},
End: &storepb.Position{Line: 103, Column: 10},
+ Range: &storepb.Range{Start: 0, End: 1080},
},
{
Text: " SELECT * FROM t;",
BaseLine: 102,
Start: &storepb.Position{Line: 103, Column: 12},
End: &storepb.Position{Line: 103, Column: 27},
+ Range: &storepb.Range{Start: 1080, End: 1097},
},
},
},
@@ -323,11 +329,13 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: `select * from t;`,
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 16},
+ Range: &storepb.Range{Start: 0, End: 16},
},
{
Text: `select "\"" where true;`,
Start: &storepb.Position{Line: 1, Column: 17},
End: &storepb.Position{Line: 1, Column: 39},
+ Range: &storepb.Range{Start: 16, End: 39},
},
},
},
@@ -343,6 +351,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Start: &storepb.Position{Line: 3, Column: 1},
End: &storepb.Position{Line: 2, Column: 23},
Empty: true,
+ Range: &storepb.Range{Start: 0, End: 39},
},
},
},
@@ -356,6 +365,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: `select * from t;`,
End: &storepb.Position{Line: 1, Column: 16},
Start: &storepb.Position{Line: 1, Column: 1},
+ Range: &storepb.Range{Start: 0, End: 16},
},
{
Text: "\n\t\t\t/* sdfasdf */",
@@ -363,6 +373,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
End: &storepb.Position{Line: 2, Column: 4},
Start: &storepb.Position{Line: 2, Column: 17},
Empty: true,
+ Range: &storepb.Range{Start: 16, End: 33},
},
},
},
@@ -377,18 +388,21 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: `select * from t;`,
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 16},
+ Range: &storepb.Range{Start: 0, End: 16},
},
{
Text: "\n\t\t\t/* sdfasdf */;",
End: &storepb.Position{Line: 2, Column: 17},
Start: &storepb.Position{Line: 2, Column: 17},
Empty: true,
+ Range: &storepb.Range{Start: 16, End: 34},
},
{
Text: "\n\t\t\tselect * from t;",
BaseLine: 1,
End: &storepb.Position{Line: 3, Column: 19},
Start: &storepb.Position{Line: 3, Column: 4},
+ Range: &storepb.Range{Start: 34, End: 54},
},
},
},
@@ -427,6 +441,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
END ;`,
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 14, Column: 7},
+ Range: &storepb.Range{Start: 0, End: 268},
},
},
},
@@ -439,6 +454,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: bigSQL,
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: int32(len(bigSQL))},
+ Range: &storepb.Range{Start: 0, End: int32(len(bigSQL))},
},
},
},
@@ -451,11 +467,13 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: " CREATE TABLE t(a int);",
Start: &storepb.Position{Line: 1, Column: 5},
End: &storepb.Position{Line: 1, Column: 26},
+ Range: &storepb.Range{Start: 0, End: 26},
},
{
Text: " CREATE TABLE t1(a int)",
Start: &storepb.Position{Line: 1, Column: 28},
End: &storepb.Position{Line: 1, Column: 49},
+ Range: &storepb.Range{Start: 26, End: 49},
},
},
},
@@ -469,11 +487,13 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: "CREATE TABLE `tech_Book`(id int, name varchar(255));",
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 52},
+ Range: &storepb.Range{Start: 0, End: 52},
},
{
Text: "\nINSERT INTO `tech_Book` VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');",
Start: &storepb.Position{Line: 2, Column: 1},
End: &storepb.Position{Line: 2, Column: 78},
+ Range: &storepb.Range{Start: 52, End: 131},
},
},
},
@@ -492,18 +512,21 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE tech_Book(id int, name varchar(255));",
Start: &storepb.Position{Line: 3, Column: 7},
End: &storepb.Position{Line: 3, Column: 76},
+ Range: &storepb.Range{Start: 0, End: 110},
},
{
Text: "\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');",
Start: &storepb.Position{Line: 5, Column: 7},
BaseLine: 2,
End: &storepb.Position{Line: 5, Column: 82},
+ Range: &storepb.Range{Start: 110, End: 223},
},
{
Text: "\n\t\t\t\t\t\t# this is the comment.\n\t\t\t\t\t\tINSERT INTO tech_Book VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafa\\'jdfl;\"ka');",
BaseLine: 4,
Start: &storepb.Position{Line: 7, Column: 7},
End: &storepb.Position{Line: 7, Column: 82},
+ Range: &storepb.Range{Start: 223, End: 335},
},
},
},
@@ -525,18 +548,21 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;",
Start: &storepb.Position{Line: 2, Column: 7},
End: &storepb.Position{Line: 7, Column: 7},
+ Range: &storepb.Range{Start: 0, End: 181},
},
{
Text: "\n\t\t\t\t\t\tCALL dorepeat(1000);",
BaseLine: 6,
Start: &storepb.Position{Line: 8, Column: 7},
End: &storepb.Position{Line: 8, Column: 26},
+ Range: &storepb.Range{Start: 181, End: 208},
},
{
Text: "\n\t\t\t\t\t\tSELECT @x;",
BaseLine: 7,
Start: &storepb.Position{Line: 9, Column: 7},
End: &storepb.Position{Line: 9, Column: 16},
+ Range: &storepb.Range{Start: 208, End: 225},
},
{
Text: "\n\t\t\t\t\t\t",
@@ -545,6 +571,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
// TODO(zp): Wait, but why the start position is larger than the end position?
End: &storepb.Position{Line: 10, Column: 6},
Empty: true,
+ Range: &storepb.Range{Start: 225, End: 232},
},
},
},
@@ -567,18 +594,21 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Text: "# test for defining stored programs\n\t\t\t\t\t\tCREATE PROCEDURE dorepeat(p1 INT)\n\t\t\t\t\t\t/* This is a comment */\n\t\t\t\t\t\tBEGIN\n\t\t\t\t\t\t\tSET @x = 0;\n\t\t\t\t\t\t\tREPEAT SET @x = @x + 1; UNTIL @x > p1 END REPEAT;\n\t\t\t\t\t\tEND\n\t\t\t\t\t\t;",
End: &storepb.Position{Line: 8, Column: 7},
Start: &storepb.Position{Line: 2, Column: 7},
+ Range: &storepb.Range{Start: 0, End: 211},
},
{
Text: "\n\t\t\t\t\t\tCALL dorepeat(1000);",
BaseLine: 7,
Start: &storepb.Position{Line: 9, Column: 7},
End: &storepb.Position{Line: 9, Column: 26},
+ Range: &storepb.Range{Start: 211, End: 238},
},
{
Text: "\n\t\t\t\t\t\tSELECT @x;",
BaseLine: 8,
Start: &storepb.Position{Line: 10, Column: 7},
End: &storepb.Position{Line: 10, Column: 16},
+ Range: &storepb.Range{Start: 238, End: 255},
},
{
Text: "\n\t\t\t\t\t\t",
@@ -586,6 +616,7 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
Start: &storepb.Position{Line: 11, Column: 7},
End: &storepb.Position{Line: 11, Column: 6},
Empty: true,
+ Range: &storepb.Range{Start: 255, End: 262},
},
},
},
@@ -600,12 +631,14 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
BaseLine: 0,
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 2, Column: 8},
+ Range: &storepb.Range{Start: 0, End: 24},
},
{
Text: "\r\nCREATE TABLE t1(b int);",
BaseLine: 1,
Start: &storepb.Position{Line: 3, Column: 1},
End: &storepb.Position{Line: 3, Column: 23},
+ Range: &storepb.Range{Start: 24, End: 49},
},
},
},
diff --git a/backend/plugin/parser/partiql/split_test.go b/backend/plugin/parser/partiql/split_test.go
index e324cc2768..65bce56ce2 100644
--- a/backend/plugin/parser/partiql/split_test.go
+++ b/backend/plugin/parser/partiql/split_test.go
@@ -19,6 +19,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "DELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money';",
BaseLine: 0,
+ Range: &storepb.Range{Start: 0, End: 70},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 70},
Empty: false,
@@ -26,6 +27,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money'};",
BaseLine: 0,
+ Range: &storepb.Range{Start: 70, End: 201},
Start: &storepb.Position{Line: 2, Column: 2},
End: &storepb.Position{Line: 2, Column: 130},
Empty: false,
@@ -33,6 +35,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money02'};",
BaseLine: 1,
+ Range: &storepb.Range{Start: 201, End: 334},
Start: &storepb.Position{Line: 3, Column: 2},
End: &storepb.Position{Line: 3, Column: 132},
Empty: false,
@@ -40,6 +43,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tINSERT INTO Music VALUE {'AlbumTitle': 'The Dark Side of the Moon', 'Artist': 'Pink Floyd', 'Awards': 300, 'SongTitle': 'Money03'};",
BaseLine: 2,
+ Range: &storepb.Range{Start: 334, End: 467},
Start: &storepb.Position{Line: 4, Column: 2},
End: &storepb.Position{Line: 4, Column: 132},
Empty: false,
@@ -47,6 +51,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tDELETE FROM Music WHERE Artist = 'Pink Floyd' AND SongTitle = 'Money02'",
BaseLine: 3,
+ Range: &storepb.Range{Start: 467, End: 540},
Start: &storepb.Position{Line: 5, Column: 2},
End: &storepb.Position{Line: 5, Column: 73},
Empty: false,
diff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.go
index 402887a22b..113a465fda 100644
--- a/backend/plugin/parser/pg/split.go
+++ b/backend/plugin/parser/pg/split.go
@@ -72,6 +72,10 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antl
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
Column: int32(tokens[pos].GetColumn()),
@@ -90,6 +94,10 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antl
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[eofPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
Column: int32(tokens[eofPos-1].GetColumn()),
@@ -191,6 +199,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
Column: int32(tokens[pos].GetColumn()),
@@ -209,6 +221,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[eofPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
Column: int32(tokens[eofPos-1].GetColumn()),
diff --git a/backend/plugin/parser/pg/split_test.go b/backend/plugin/parser/pg/split_test.go
index 0583521ce3..86307aee98 100644
--- a/backend/plugin/parser/pg/split_test.go
+++ b/backend/plugin/parser/pg/split_test.go
@@ -44,6 +44,7 @@ func TestPGSplitMultiSQL(t *testing.T) {
{
Text: "-- klsjdfjasldf\n\t\t\t-- klsjdflkjaskldfj\n\t\t\t",
BaseLine: 0,
+ Range: &storepb.Range{Start: 0, End: 42},
Start: &storepb.Position{Line: 3, Column: 4},
End: &storepb.Position{Line: 3, Column: 3},
Empty: true,
@@ -58,11 +59,13 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: `select * from t;`,
+ Range: &storepb.Range{Start: 0, End: 16},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 16},
},
{
Text: "\n\t\t\t/* sdfasdf */",
+ Range: &storepb.Range{Start: 16, End: 33},
End: &storepb.Position{Line: 2, Column: 4},
Start: &storepb.Position{Line: 2, Column: 17},
Empty: true,
@@ -78,11 +81,13 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: `select * from t;`,
+ Range: &storepb.Range{Start: 0, End: 16},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 16},
},
{
Text: "\n\t\t\t/* sdfasdf */;",
+ Range: &storepb.Range{Start: 16, End: 34},
Start: &storepb.Position{Line: 2, Column: 17},
End: &storepb.Position{Line: 2, Column: 17},
Empty: true,
@@ -90,6 +95,7 @@ func TestPGSplitMultiSQL(t *testing.T) {
{
Text: "\n\t\t\tselect * from t;",
BaseLine: 1,
+ Range: &storepb.Range{Start: 34, End: 54},
Start: &storepb.Position{Line: 3, Column: 4},
End: &storepb.Position{Line: 3, Column: 19},
},
@@ -102,6 +108,7 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: bigSQL,
+ Range: &storepb.Range{Start: 0, End: 1048600},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: int32(len(bigSQL))},
},
@@ -114,11 +121,13 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: " CREATE TABLE t(a int);",
+ Range: &storepb.Range{Start: 0, End: 26},
Start: &storepb.Position{Line: 1, Column: 5},
End: &storepb.Position{Line: 1, Column: 26},
},
{
Text: " CREATE TABLE t1(a int)",
+ Range: &storepb.Range{Start: 26, End: 49},
Start: &storepb.Position{Line: 1, Column: 28},
End: &storepb.Position{Line: 1, Column: 49},
},
@@ -132,11 +141,13 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: `CREATE TABLE "tech_Book"(id int, name varchar(255));`,
+ Range: &storepb.Range{Start: 0, End: 52},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 52},
},
{
Text: "\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');",
+ Range: &storepb.Range{Start: 52, End: 135},
Start: &storepb.Position{Line: 2, Column: 7},
End: &storepb.Position{Line: 2, Column: 82},
},
@@ -153,12 +164,14 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: "\n\t\t\t\t\t\t/* this is the comment. */\n\t\t\t\t\t\tCREATE /* inline comment */TABLE \"tech_Book\"(id int, name varchar(255));",
+ Range: &storepb.Range{Start: 0, End: 112},
Start: &storepb.Position{Line: 3, Column: 7},
End: &storepb.Position{Line: 3, Column: 78},
},
{
Text: "\n\t\t\t\t\t\t-- this is the comment.\n\t\t\t\t\t\tINSERT INTO \"tech_Book\" VALUES (0, 'abce_ksdf'), (1, 'lks''kjsafajdfl;\"ka');",
BaseLine: 2,
+ Range: &storepb.Range{Start: 112, End: 225},
Start: &storepb.Position{Line: 5, Column: 7},
End: &storepb.Position{Line: 5, Column: 82},
},
@@ -186,12 +199,14 @@ func TestPGSplitMultiSQL(t *testing.T) {
-- this is the comment
INSERT INTO tbl VALUES ('fasf_bkdjlfa');
$$;`,
+ Range: &storepb.Range{Start: 0, End: 303},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 8, Column: 9},
},
{
Text: "\n\t\t\t\t\t\tCREATE TABLE t(a int);",
BaseLine: 7,
+ Range: &storepb.Range{Start: 303, End: 332},
Start: &storepb.Position{Line: 9, Column: 7},
End: &storepb.Position{Line: 9, Column: 28},
},
@@ -219,12 +234,14 @@ func TestPGSplitMultiSQL(t *testing.T) {
-- this is the comment
INSERT INTO tbl VALUES ('fasf_bkdjlfa');
$tag_name$;`,
+ Range: &storepb.Range{Start: 0, End: 315},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 8, Column: 17},
},
{
Text: "\n\t\t\t\t\t\tCREATE TABLE t(a int);",
BaseLine: 7,
+ Range: &storepb.Range{Start: 315, End: 344},
Start: &storepb.Position{Line: 9, Column: 7},
End: &storepb.Position{Line: 9, Column: 28},
},
@@ -238,12 +255,14 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: "CREATE TABLE t\r\n(a int);",
+ Range: &storepb.Range{Start: 0, End: 24},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 2, Column: 8},
},
{
Text: "\r\nCREATE TABLE t1(b int);",
BaseLine: 1,
+ Range: &storepb.Range{Start: 24, End: 49},
Start: &storepb.Position{Line: 3, Column: 1},
End: &storepb.Position{Line: 3, Column: 23},
},
@@ -260,6 +279,7 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: "INSERT INTO \"public\".\"table\"(\"id\",\"content\")\n\t\t\tVALUES\n\t\t\t(12,'table column name () { :xna,sydfn,,kasdfyn;}; /////test string/// 0'),\n\t\t\t(133,'knuandfan public table id'';create table t(a int, b int);set @text=''\\\\\\\\kdaminxkljasdfiebkla.unkonwn''+''abcdef.xyz\\\\''; local xxxyy.abcddd.mysql @text;------- '),\n\t\t\t(1444,'table t xyz abc a''a\\\\\\\\\\\\\\\\''b\"c>?>xxxxxx%}}%%>c<[[?${12344556778990{%}}cake\\\\');",
+ Range: &storepb.Range{Start: 0, End: 400},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 5, Column: 92},
},
@@ -272,6 +292,7 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: "INSERT INTO t VALUES ('klajfas)",
+ Range: &storepb.Range{Start: 0, End: 31},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 23},
Empty: false,
@@ -285,6 +306,7 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: "INSERT INTO \"t VALUES ('klajfas)",
+ Range: &storepb.Range{Start: 0, End: 32},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 13},
},
@@ -298,6 +320,7 @@ func TestPGSplitMultiSQL(t *testing.T) {
{
Text: "/*INSERT INTO \"t VALUES ('klajfas)",
BaseLine: 0,
+ Range: &storepb.Range{Start: 0, End: 34},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 1},
Empty: false,
@@ -311,6 +334,7 @@ func TestPGSplitMultiSQL(t *testing.T) {
res: []base.Statement{
{
Text: "$$INSERT INTO \"t VALUES ('klajfas)",
+ Range: &storepb.Range{Start: 0, End: 34},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 3},
},
diff --git a/backend/plugin/parser/plsql/split.go b/backend/plugin/parser/plsql/split.go
index ec00684061..edfc1a600c 100644
--- a/backend/plugin/parser/plsql/split.go
+++ b/backend/plugin/parser/plsql/split.go
@@ -85,9 +85,11 @@ func SplitSQL(statement string) ([]base.Statement, error) {
},
statement,
),
- Empty: base.IsEmpty(tokens.GetAllTokens()[stmt.GetStart().GetTokenIndex():stmt.GetStop().GetTokenIndex()+1], parser.PlSqlParserSEMICOLON),
- ByteOffsetStart: byteOffsetStart,
- ByteOffsetEnd: byteOffsetEnd,
+ Empty: base.IsEmpty(tokens.GetAllTokens()[stmt.GetStart().GetTokenIndex():stmt.GetStop().GetTokenIndex()+1], parser.PlSqlParserSEMICOLON),
+ Range: &storepb.Range{
+ Start: int32(byteOffsetStart),
+ End: int32(byteOffsetEnd),
+ },
})
byteOffsetStart = byteOffsetEnd
prevStopTokenIndex = stmt.GetStop().GetTokenIndex()
diff --git a/backend/plugin/parser/plsql/split_test.go b/backend/plugin/parser/plsql/split_test.go
index 973e2e1d22..c79823801d 100644
--- a/backend/plugin/parser/plsql/split_test.go
+++ b/backend/plugin/parser/plsql/split_test.go
@@ -32,20 +32,20 @@ func TestOracleSplitMultiSQL(t *testing.T) {
want: resData{
res: []base.Statement{
{
- Text: `select * from t`,
- BaseLine: 1,
- ByteOffsetStart: 5,
- ByteOffsetEnd: 20,
- Start: &storepb.Position{Line: 2, Column: 5},
- End: &storepb.Position{Line: 2, Column: 19},
+ Text: `select * from t`,
+ BaseLine: 1,
+ Range: &storepb.Range{Start: int32(5),
+ End: int32(20)},
+ Start: &storepb.Position{Line: 2, Column: 5},
+ End: &storepb.Position{Line: 2, Column: 19},
},
{
- Text: `create table table$1 (id int)`,
- BaseLine: 2,
- ByteOffsetStart: 26,
- ByteOffsetEnd: 56,
- Start: &storepb.Position{Line: 3, Column: 5},
- End: &storepb.Position{Line: 3, Column: 33},
+ Text: `create table table$1 (id int)`,
+ BaseLine: 2,
+ Range: &storepb.Range{Start: int32(26),
+ End: int32(56)},
+ Start: &storepb.Position{Line: 3, Column: 5},
+ End: &storepb.Position{Line: 3, Column: 33},
},
},
},
@@ -63,10 +63,10 @@ ONLINE;`,
want: resData{
res: []base.Statement{
{
- Text: "ALTER TABLE DATA.TEST\nMODIFY PARTITION BY RANGE (TXN_DATE)\nINTERVAL (NUMTODSINTERVAL(1, 'DAY'))\n(\n\tPARTITION TEST_PO VALUES LESS THAN (\n\t\tTO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')\n\t)\n)\nONLINE",
- Start: &storepb.Position{Line: 1, Column: 1},
- End: &storepb.Position{Line: 9, Column: 1},
- ByteOffsetEnd: 233,
+ Text: "ALTER TABLE DATA.TEST\nMODIFY PARTITION BY RANGE (TXN_DATE)\nINTERVAL (NUMTODSINTERVAL(1, 'DAY'))\n(\n\tPARTITION TEST_PO VALUES LESS THAN (\n\t\tTO_DATE('2000-01-01 00:00:00', 'SYYYY-MM-DD HH24:MI:SS', 'NLS_CALENDAR=GREGORIAN')\n\t)\n)\nONLINE",
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 9, Column: 1},
+ Range: &storepb.Range{Start: 0, End: 233},
},
},
},
@@ -90,10 +90,10 @@ ONLINE;`,
want: resData{
res: []base.Statement{
{
- Text: "DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS",
- Start: &storepb.Position{Line: 1, Column: 1},
- End: &storepb.Position{Line: 1, Column: 48},
- ByteOffsetEnd: 58,
+ Text: "DROP TABLESPACE xxx INCLUDING CONTENTS CASCADE CONSTRAINTS",
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 48},
+ Range: &storepb.Range{Start: 0, End: 58},
},
},
},
@@ -103,10 +103,10 @@ ONLINE;`,
want: resData{
res: []base.Statement{
{
- Text: "CALL my_procedure()",
- Start: &storepb.Position{Line: 1, Column: 1},
- End: &storepb.Position{Line: 1, Column: 19},
- ByteOffsetEnd: 19,
+ Text: "CALL my_procedure()",
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 19},
+ Range: &storepb.Range{Start: 0, End: 19},
},
},
},
@@ -116,18 +116,18 @@ ONLINE;`,
want: resData{
res: []base.Statement{
{
- Text: "SELECT * FROM t1",
- Start: &storepb.Position{Line: 1, Column: 1},
- End: &storepb.Position{Line: 1, Column: 15},
- ByteOffsetStart: 0,
- ByteOffsetEnd: 16,
+ Text: "SELECT * FROM t1",
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 15},
+ Range: &storepb.Range{Start: int32(0),
+ End: int32(16)},
},
{
- Text: "SELECT * FROM t2",
- Start: &storepb.Position{Line: 1, Column: 19},
- End: &storepb.Position{Line: 1, Column: 33},
- ByteOffsetStart: 18,
- ByteOffsetEnd: 34,
+ Text: "SELECT * FROM t2",
+ Start: &storepb.Position{Line: 1, Column: 19},
+ End: &storepb.Position{Line: 1, Column: 33},
+ Range: &storepb.Range{Start: int32(18),
+ End: int32(34)},
},
},
},
@@ -142,10 +142,10 @@ END;
want: resData{
res: []base.Statement{
{
- Text: "CREATE OR REPLACE PROCEDURE test_proc IS\nBEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Hello');\nEND;",
- Start: &storepb.Position{Line: 1, Column: 1},
- End: &storepb.Position{Line: 4, Column: 4},
- ByteOffsetEnd: 91,
+ Text: "CREATE OR REPLACE PROCEDURE test_proc IS\nBEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Hello');\nEND;",
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 4, Column: 4},
+ Range: &storepb.Range{Start: 0, End: 91},
},
},
},
@@ -164,20 +164,20 @@ END;
want: resData{
res: []base.Statement{
{
- Text: "CREATE OR REPLACE PROCEDURE proc1 IS\nBEGIN\n NULL;\nEND;",
- BaseLine: 0,
- Start: &storepb.Position{Line: 1, Column: 1},
- End: &storepb.Position{Line: 4, Column: 4},
- ByteOffsetStart: 0,
- ByteOffsetEnd: 57,
+ Text: "CREATE OR REPLACE PROCEDURE proc1 IS\nBEGIN\n NULL;\nEND;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 4, Column: 4},
+ Range: &storepb.Range{Start: int32(0),
+ End: int32(57)},
},
{
- Text: "CREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;",
- BaseLine: 5,
- Start: &storepb.Position{Line: 6, Column: 1},
- End: &storepb.Position{Line: 9, Column: 4},
- ByteOffsetStart: 60,
- ByteOffsetEnd: 117,
+ Text: "CREATE OR REPLACE PROCEDURE proc2 IS\nBEGIN\n NULL;\nEND;",
+ BaseLine: 5,
+ Start: &storepb.Position{Line: 6, Column: 1},
+ End: &storepb.Position{Line: 9, Column: 4},
+ Range: &storepb.Range{Start: int32(60),
+ End: int32(117)},
},
},
},
@@ -189,20 +189,20 @@ SELECT * FROM t2;`,
want: resData{
res: []base.Statement{
{
- Text: "SELECT * FROM t1",
- BaseLine: 0,
- Start: &storepb.Position{Line: 1, Column: 1},
- End: &storepb.Position{Line: 1, Column: 15},
- ByteOffsetStart: 0,
- ByteOffsetEnd: 16,
+ Text: "SELECT * FROM t1",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 15},
+ Range: &storepb.Range{Start: int32(0),
+ End: int32(16)},
},
{
- Text: "SELECT * FROM t2",
- BaseLine: 2,
- Start: &storepb.Position{Line: 3, Column: 1},
- End: &storepb.Position{Line: 3, Column: 15},
- ByteOffsetStart: 20,
- ByteOffsetEnd: 36,
+ Text: "SELECT * FROM t2",
+ BaseLine: 2,
+ Start: &storepb.Position{Line: 3, Column: 1},
+ End: &storepb.Position{Line: 3, Column: 15},
+ Range: &storepb.Range{Start: int32(20),
+ End: int32(36)},
},
},
},
@@ -215,10 +215,10 @@ END;
want: resData{
res: []base.Statement{
{
- Text: "BEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Test');\nEND;",
- Start: &storepb.Position{Line: 1, Column: 1},
- End: &storepb.Position{Line: 3, Column: 4},
- ByteOffsetEnd: 49,
+ Text: "BEGIN\n CALL DBMS_OUTPUT.PUT_LINE('Test');\nEND;",
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 3, Column: 4},
+ Range: &storepb.Range{Start: 0, End: 49},
},
},
},
diff --git a/backend/plugin/parser/redshift/split.go b/backend/plugin/parser/redshift/split.go
index 1373c1c7d3..521d6e7043 100644
--- a/backend/plugin/parser/redshift/split.go
+++ b/backend/plugin/parser/redshift/split.go
@@ -73,6 +73,10 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
Column: int32(tokens[pos].GetColumn()),
@@ -91,6 +95,10 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[eofPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
Column: int32(tokens[eofPos-1].GetColumn()),
@@ -191,6 +199,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
Column: int32(tokens[pos].GetColumn()),
@@ -209,6 +221,10 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[eofPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
Column: int32(tokens[eofPos-1].GetColumn()),
diff --git a/backend/plugin/parser/snowflake/split_test.go b/backend/plugin/parser/snowflake/split_test.go
index a12282729f..7a87ab2cd6 100644
--- a/backend/plugin/parser/snowflake/split_test.go
+++ b/backend/plugin/parser/snowflake/split_test.go
@@ -17,6 +17,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "SELECT * FROM users;",
BaseLine: 0,
+ Range: &storepb.Range{Start: 0, End: 20},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 20},
Empty: false,
@@ -24,6 +25,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tSELECT * FROM orders;",
BaseLine: 0,
+ Range: &storepb.Range{Start: 20, End: 43},
Start: &storepb.Position{Line: 2, Column: 2},
End: &storepb.Position{Line: 2, Column: 22},
Empty: false,
@@ -31,6 +33,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tSELECT * FROM products;",
BaseLine: 1,
+ Range: &storepb.Range{Start: 43, End: 68},
Start: &storepb.Position{Line: 3, Column: 2},
End: &storepb.Position{Line: 3, Column: 24},
Empty: false,
diff --git a/backend/plugin/parser/spanner/split.go b/backend/plugin/parser/spanner/split.go
index b6f586b419..4068b0964f 100644
--- a/backend/plugin/parser/spanner/split.go
+++ b/backend/plugin/parser/spanner/split.go
@@ -92,6 +92,10 @@ func SplitSQL(statement string) ([]base.Statement, error) {
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[endPos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[endPos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[endPos].GetLine()),
Column: int32(tokens[endPos].GetColumn()),
diff --git a/backend/plugin/parser/spanner/split_test.go b/backend/plugin/parser/spanner/split_test.go
index 260ba442fb..02f738dc1e 100644
--- a/backend/plugin/parser/spanner/split_test.go
+++ b/backend/plugin/parser/spanner/split_test.go
@@ -17,6 +17,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "SELECT * FROM users;",
BaseLine: 0,
+ Range: &storepb.Range{Start: 0, End: 20},
Start: &storepb.Position{Line: 1, Column: 1},
End: &storepb.Position{Line: 1, Column: 20},
Empty: false,
@@ -24,6 +25,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tSELECT * FROM orders;",
BaseLine: 0,
+ Range: &storepb.Range{Start: 20, End: 43},
Start: &storepb.Position{Line: 2, Column: 2},
End: &storepb.Position{Line: 2, Column: 22},
Empty: false,
@@ -31,6 +33,7 @@ func TestSplitSQL(t *testing.T) {
{
Text: "\n\tSELECT * FROM products;",
BaseLine: 1,
+ Range: &storepb.Range{Start: 43, End: 68},
Start: &storepb.Position{Line: 3, Column: 2},
End: &storepb.Position{Line: 3, Column: 24},
Empty: false,
diff --git a/backend/plugin/parser/standard/split.go b/backend/plugin/parser/standard/split.go
index 756f9cd57d..08b308d6f6 100644
--- a/backend/plugin/parser/standard/split.go
+++ b/backend/plugin/parser/standard/split.go
@@ -23,12 +23,27 @@ func init() {
// SplitSQL splits the given SQL statement into multiple SQL statements.
func SplitSQL(statement string) ([]base.Statement, error) {
var list []base.Statement
+ byteOffset := 0
err := applyMultiStatements(strings.NewReader(statement), func(sql string) error {
+ // Find the start position of this SQL in the original statement
+ startPos := strings.Index(statement[byteOffset:], strings.TrimLeft(sql, "\n\t "))
+ if startPos != -1 {
+ startPos += byteOffset
+ } else {
+ startPos = byteOffset
+ }
+ endPos := startPos + len(sql)
+
list = append(list, base.Statement{
- Text: sql,
+ Text: sql,
+ Range: &storepb.Range{
+ Start: int32(startPos),
+ End: int32(endPos),
+ },
End: &storepb.Position{Line: 0, Column: 0},
Empty: false,
})
+ byteOffset = endPos
return nil
})
return list, err
diff --git a/backend/plugin/parser/tidb/split.go b/backend/plugin/parser/tidb/split.go
index abf0164d24..a940a93a5f 100644
--- a/backend/plugin/parser/tidb/split.go
+++ b/backend/plugin/parser/tidb/split.go
@@ -97,6 +97,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[i]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[i].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[i].GetLine()),
@@ -125,6 +129,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
// Use a single semicolon instead of the user defined delimiter.
Text: stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";",
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[newStart-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[newStart-1].GetLine()),
@@ -151,6 +159,10 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[endPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[endPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[endPos-1].GetLine()),
@@ -315,6 +327,10 @@ func splitTiDBStatement(stream *antlr.CommonTokenStream, statement string) ([]ba
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -336,6 +352,10 @@ func splitTiDBStatement(stream *antlr.CommonTokenStream, statement string) ([]ba
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[eofPos-1].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
diff --git a/backend/plugin/parser/tokenizer/tokenizer.go b/backend/plugin/parser/tokenizer/tokenizer.go
index 146d02ee8e..98477a1fbc 100644
--- a/backend/plugin/parser/tokenizer/tokenizer.go
+++ b/backend/plugin/parser/tokenizer/tokenizer.go
@@ -246,10 +246,12 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
// but we want to get the line of last line of the SQL
// which means the line of ')'.
// So we need minus the aboveNonBlankLineDistance.
- End: &store.Position{Line: int32(t.line - t.aboveNonBlankLineDistance())},
- Empty: t.emptyStatement,
- ByteOffsetStart: t.getByteOffset(int(startPos)),
- ByteOffsetEnd: t.getByteOffset(int(t.pos())),
+ End: &store.Position{Line: int32(t.line - t.aboveNonBlankLineDistance())},
+ Empty: t.emptyStatement,
+ Range: &store.Range{
+ Start: int32(t.getByteOffset(int(startPos))),
+ End: int32(t.getByteOffset(int(t.pos()))),
+ },
})
}
if err := t.processStreaming(s); err != nil {
@@ -265,11 +267,13 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
res = append(res, base.Statement{
Text: text,
// BaseLine is 0-based line number, so subtract 1 from 1-based startLine
- BaseLine: startLine - 1,
- End: &store.Position{Line: int32(t.line)},
- Empty: t.emptyStatement,
- ByteOffsetStart: t.getByteOffset(int(startPos)),
- ByteOffsetEnd: t.getByteOffset(int(t.pos())),
+ BaseLine: startLine - 1,
+ End: &store.Position{Line: int32(t.line)},
+ Empty: t.emptyStatement,
+ Range: &store.Range{
+ Start: int32(t.getByteOffset(int(startPos))),
+ End: int32(t.getByteOffset(int(t.pos()))),
+ },
})
}
t.skipBlank()
@@ -291,11 +295,13 @@ func (t *Tokenizer) SplitTiDBMultiSQL() ([]base.Statement, error) {
res = append(res, base.Statement{
Text: text,
// BaseLine is 0-based line number, so subtract 1 from 1-based startLine
- BaseLine: startLine - 1,
- End: &store.Position{Line: int32(t.line)},
- Empty: false,
- ByteOffsetStart: t.getByteOffset(int(startPos)),
- ByteOffsetEnd: t.getByteOffset(int(t.pos())),
+ BaseLine: startLine - 1,
+ End: &store.Position{Line: int32(t.line)},
+ Empty: false,
+ Range: &store.Range{
+ Start: int32(t.getByteOffset(int(startPos))),
+ End: int32(t.getByteOffset(int(t.pos()))),
+ },
})
}
t.skipBlank()
@@ -398,6 +404,10 @@ func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {
Column: int32(firstStatementColumn),
},
Empty: t.emptyStatement,
+ Range: &store.Range{
+ Start: int32(t.getByteOffset(int(startPos))),
+ End: int32(t.getByteOffset(int(t.pos()))),
+ },
})
}
t.skipBlank()
@@ -433,6 +443,10 @@ func (t *Tokenizer) SplitStandardMultiSQL() ([]base.Statement, error) {
Column: int32(firstStatementColumn),
},
Empty: t.emptyStatement,
+ Range: &store.Range{
+ Start: int32(t.getByteOffset(int(startPos))),
+ End: int32(t.getByteOffset(int(t.pos()))),
+ },
})
}
if err := t.processStreaming(s); err != nil {
@@ -510,6 +524,10 @@ func (t *Tokenizer) SplitPostgreSQLMultiSQL() ([]base.Statement, error) {
BaseLine: startLine - 1,
End: &store.Position{Line: int32(t.line)},
Empty: t.emptyStatement,
+ Range: &store.Range{
+ Start: int32(t.getByteOffset(int(startPos))),
+ End: int32(t.getByteOffset(int(t.pos()))),
+ },
})
}
t.skipBlank()
@@ -543,6 +561,10 @@ func (t *Tokenizer) SplitPostgreSQLMultiSQL() ([]base.Statement, error) {
Line: int32(t.line - t.aboveNonBlankLineDistance()),
},
Empty: t.emptyStatement,
+ Range: &store.Range{
+ Start: int32(t.getByteOffset(int(startPos))),
+ End: int32(t.getByteOffset(int(t.pos()))),
+ },
})
}
if err := t.processStreaming(s); err != nil {
diff --git a/backend/plugin/parser/trino/split.go b/backend/plugin/parser/trino/split.go
index ab5d47acb9..3727ef128a 100644
--- a/backend/plugin/parser/trino/split.go
+++ b/backend/plugin/parser/trino/split.go
@@ -191,6 +191,10 @@ func splitByParser(statement string) ([]base.Statement, error) {
result = append(result, base.Statement{
Text: text,
BaseLine: firstToken.GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[startIdx].GetStart()),
+ End: int32(endToken.GetStop() + 1),
+ },
Start: &storepb.Position{
Line: int32(firstDefaultToken.GetLine() - 1),
Column: int32(firstDefaultToken.GetColumn()),
diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.go
index c2f0e5cd04..d8f8daf6c4 100644
--- a/backend/plugin/parser/tsql/split.go
+++ b/backend/plugin/parser/tsql/split.go
@@ -76,6 +76,10 @@ func splitByParser(statement string) ([]base.Statement, error) {
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -118,6 +122,10 @@ func splitByParser(statement string) ([]base.Statement, error) {
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -144,6 +152,10 @@ func splitByParser(statement string) ([]base.Statement, error) {
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -172,6 +184,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -190,6 +206,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -207,6 +227,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -224,6 +248,10 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
result = append(result, base.Statement{
Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
BaseLine: tokens[start].GetLine() - 1,
+ Range: &storepb.Range{
+ Start: int32(tokens[start].GetStart()),
+ End: int32(tokens[pos].GetStop() + 1),
+ },
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
diff --git a/backend/plugin/parser/tsql/split_test.go b/backend/plugin/parser/tsql/split_test.go
index 9a4028ac14..38b218d925 100644
--- a/backend/plugin/parser/tsql/split_test.go
+++ b/backend/plugin/parser/tsql/split_test.go
@@ -43,6 +43,7 @@ go
Start: &storepb.Position{Line: 2, Column: 1},
End: &storepb.Position{Line: 2, Column: 26},
Empty: false,
+ Range: &storepb.Range{Start: 0, End: 45},
},
{
Text: "\n-- second statement\ninsert into @temp values(1)",
@@ -50,6 +51,7 @@ go
Start: &storepb.Position{Line: 4, Column: 1},
End: &storepb.Position{Line: 4, Column: 27},
Empty: false,
+ Range: &storepb.Range{Start: 45, End: 93},
},
{
Text: "\n-- third statement\nselect * from @temp\n-- go statement\ngo",
@@ -57,6 +59,7 @@ go
Start: &storepb.Position{Line: 6, Column: 1},
End: &storepb.Position{Line: 8, Column: 1},
Empty: false,
+ Range: &storepb.Range{Start: 93, End: 151},
},
{
Text: "\ngo",
@@ -64,6 +67,7 @@ go
Start: &storepb.Position{Line: 9, Column: 1},
End: &storepb.Position{Line: 9, Column: 1},
Empty: false,
+ Range: &storepb.Range{Start: 151, End: 154},
},
{
Text: "\ngo\ngo",
@@ -71,6 +75,7 @@ go
Start: &storepb.Position{Line: 10, Column: 1},
End: &storepb.Position{Line: 11, Column: 1},
Empty: false,
+ Range: &storepb.Range{Start: 154, End: 160},
},
},
},
@@ -103,6 +108,7 @@ UPDATE SalesLT.ProductModelProductDescription SET Culture = "zh-cn";
Start: &storepb.Position{Line: 10, Column: 1},
End: &storepb.Position{Line: 10, Column: 45},
Empty: false,
+ Range: &storepb.Range{Start: 0, End: 54},
},
{
Text: "\n\nUPDATE SalesLT.Address SET PostalCode = 0;",
@@ -110,6 +116,7 @@ UPDATE SalesLT.ProductModelProductDescription SET Culture = "zh-cn";
Start: &storepb.Position{Line: 12, Column: 1},
End: &storepb.Position{Line: 12, Column: 42},
Empty: false,
+ Range: &storepb.Range{Start: 54, End: 98},
},
{
Text: "\n\n\nUPDATE SalesLT.ProductModelProductDescription SET Culture = \"zh-cn\";",
@@ -117,6 +124,7 @@ UPDATE SalesLT.ProductModelProductDescription SET Culture = "zh-cn";
Start: &storepb.Position{Line: 15, Column: 1},
End: &storepb.Position{Line: 15, Column: 68},
Empty: false,
+ Range: &storepb.Range{Start: 98, End: 169},
},
},
},