Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat: use byte offsets for statement ranges to support UTF-8 correctly (#18536)
Danny Xubf7e827b2cb3
diff --git a/backend/plugin/parser/bigquery/split.go b/backend/plugin/parser/bigquery/split.goindex b6da131f29..71aa0e828d 100644--- a/backend/plugin/parser/bigquery/split.go+++ b/backend/plugin/parser/bigquery/split.go@@ -23,6 +23,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {tokens := stream.GetAllTokens()var buf []antlr.Tokenvar sqls []base.Statement+ byteOffset := 0for i, token := range tokens {if i < len(tokens)-1 {buf = append(buf, token)@@ -39,13 +40,15 @@ func SplitSQL(statement string) ([]base.Statement, error) {empty = false}}+ stmtText := bufStr.String()+ stmtByteLength := len(stmtText)antlrPosition := base.FirstDefaultChannelTokenPosition(buf)sqls = append(sqls, base.Statement{- Text: bufStr.String(),+ Text: stmtText,BaseLine: buf[0].GetLine() - 1,Range: &storepb.Range{- Start: int32(buf[0].GetStart()),- End: int32(buf[len(buf)-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(buf[len(buf)-1].GetLine()),@@ -54,6 +57,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: empty,})+ byteOffset += stmtByteLengthbuf = nilcontinue}diff --git a/backend/plugin/parser/mysql/split.go b/backend/plugin/parser/mysql/split.goindex eebf948c90..e8b86781de 100644--- a/backend/plugin/parser/mysql/split.go+++ b/backend/plugin/parser/mysql/split.go@@ -56,15 +56,25 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([// Deal with normal statement.if delimiter == ";" && token.GetTokenType() == parser.MySQLLexerSEMICOLON_SYMBOL {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[i])++ // Calculate byte range by getting actual bytes from original statement+ // This works because for ASCII chars, char offset == byte offset+ // For UTF-8, we get the actual substring which handles multi-byte chars correctly+ stmtStartChar := tokens[start].GetStart()+ stmtEndChar := tokens[i].GetStop() + 1+ stmtStartByte := len(statement[:stmtStartChar])+ stmtEndByte := len(statement[:stmtEndChar])+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : i+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[i]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[i].GetStop() + 1),+ Start: int32(stmtStartByte),+ End: int32(stmtEndByte),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[i].GetLine()),@@ -84,16 +94,24 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([}if newStart, ok := tryMatchDelimiter(stream, i, delimiter); ok {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";"++ // Calculate byte range by getting actual bytes from original statement+ stmtStartChar := tokens[start].GetStart()+ stmtEndChar := tokens[newStart-1].GetStop() + 1+ stmtStartByte := len(statement[:stmtStartChar])+ stmtEndByte := len(statement[:stmtEndChar])+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:newStart])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{// Use a single semicolon instead of the user defined delimiter.- Text: stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";",+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[newStart-1].GetStop() + 1),+ Start: int32(stmtStartByte),+ End: int32(stmtEndByte),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[newStart-1].GetLine()),@@ -112,15 +130,23 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([endPos := len(tokens) - 1if start < endPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[endPos-1])++ // Calculate byte range by getting actual bytes from original statement+ stmtStartChar := tokens[start].GetStart()+ stmtEndChar := tokens[endPos-1].GetStop() + 1+ stmtStartByte := len(statement[:stmtStartChar])+ stmtEndByte := len(statement[:stmtEndChar])+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[endPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[endPos-1].GetStop() + 1),+ Start: int32(stmtStartByte),+ End: int32(stmtEndByte),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[endPos-1].GetLine()),@@ -205,18 +231,22 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Comvar result []base.Statementtokens := stream.GetAllTokens()+ byteOffset := 0start := 0for _, semicolon := range tree.AllSEMICOLON_SYMBOL() {pos := semicolon.GetSymbol().GetStart()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -225,20 +255,24 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.ComStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),})+ byteOffset += stmtByteLengthstart = pos + 1}// For the last statement, it may not end with semicolon symbol, EOF symbol instead.eofPos := len(tokens) - 1if start < eofPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[eofPos-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),@@ -392,17 +426,21 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b}}+ byteOffset := 0start := 0for _, pos := range semicolonStack {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -411,20 +449,24 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]bStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),})+ byteOffset += stmtByteLengthstart = pos + 1}// For the last statement, it may not end with semicolon symbol, EOF symbol instead.eofPos := len(tokens) - 1if start < eofPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[eofPos-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),diff --git a/backend/plugin/parser/mysql/split_test.go b/backend/plugin/parser/mysql/split_test.goindex 67e132b159..afc9356988 100644--- a/backend/plugin/parser/mysql/split_test.go+++ b/backend/plugin/parser/mysql/split_test.go@@ -643,6 +643,25 @@ func TestMySQLSplitMultiSQL(t *testing.T) {},},},+ {+ statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",+ want: resData{+ res: []base.Statement{+ {+ Text: "SELECT * FROM 表名;",+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 17},+ },+ {+ Text: " INSERT INTO 表 VALUES (1);",+ Range: &storepb.Range{Start: 21, End: 49}, // Byte offset 21-49 (not 17-43)+ Start: &storepb.Position{Line: 1, Column: 19},+ End: &storepb.Position{Line: 1, Column: 43},+ },+ },+ },+ },}for _, test := range tests {diff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.goindex 113a465fda..23ac44ca5a 100644--- a/backend/plugin/parser/pg/split.go+++ b/backend/plugin/parser/pg/split.go@@ -63,18 +63,22 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlvar result []base.Statementtokens := stream.GetAllTokens()+ byteOffset := 0start := 0for _, semi := range tree.Stmtblock().Stmtmulti().AllSEMI() {pos := semi.GetSymbol().GetStart()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -83,20 +87,24 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antlStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),})+ byteOffset += stmtByteLengthstart = pos + 1}// For the last statement, it may not end with semicolon symbol, EOF symbol instead.eofPos := len(tokens) - 1if start < eofPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[eofPos-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),@@ -191,17 +199,21 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta}}+ byteOffset := 0start := 0for _, pos := range semicolonStack {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -210,20 +222,24 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.StaStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),})+ byteOffset += stmtByteLengthstart = pos + 1}// For the last statement, it may not end with semicolon symbol, EOF symbol instead.eofPos := len(tokens) - 1if start < eofPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[eofPos-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),diff --git a/backend/plugin/parser/pg/split_test.go b/backend/plugin/parser/pg/split_test.goindex 86307aee98..a9a1480cf6 100644--- a/backend/plugin/parser/pg/split_test.go+++ b/backend/plugin/parser/pg/split_test.go@@ -341,6 +341,25 @@ func TestPGSplitMultiSQL(t *testing.T) {},},},+ {+ statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",+ want: resData{+ res: []base.Statement{+ {+ Text: "SELECT * FROM 表名;",+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 17},+ },+ {+ Text: " INSERT INTO 表 VALUES (1);",+ Range: &storepb.Range{Start: 21, End: 49}, // Byte offset 21-49 (not 17-43)+ Start: &storepb.Position{Line: 1, Column: 19},+ End: &storepb.Position{Line: 1, Column: 43},+ },+ },+ },+ },}for _, test := range tests {diff --git a/backend/plugin/parser/redshift/split.go b/backend/plugin/parser/redshift/split.goindex 521d6e7043..cb4172db5c 100644--- a/backend/plugin/parser/redshift/split.go+++ b/backend/plugin/parser/redshift/split.go@@ -64,18 +64,22 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.var result []base.Statementtokens := stream.GetAllTokens()+ byteOffset := 0start := 0for _, semi := range tree.Stmtblock().Stmtmulti().AllSEMI() {pos := semi.GetSymbol().GetStart()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -84,20 +88,24 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:pos+1], parser.RedshiftParserSEMI),})+ byteOffset += stmtByteLengthstart = pos + 1}// For the last statement, it may not end with semicolon symbol, EOF symbol instead.eofPos := len(tokens) - 1if start < eofPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[eofPos-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),@@ -191,17 +199,21 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta}}+ byteOffset := 0start := 0for _, pos := range semicolonStack {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[pos].GetLine()),@@ -210,20 +222,24 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.StaStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:pos+1], parser.RedshiftParserSEMI),})+ byteOffset += stmtByteLengthstart = pos + 1}// For the last statement, it may not end with semicolon symbol, EOF symbol instead.eofPos := len(tokens) - 1if start < eofPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[eofPos-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[eofPos-1].GetLine()),diff --git a/backend/plugin/parser/spanner/split.go b/backend/plugin/parser/spanner/split.goindex 4068b0964f..df592d009b 100644--- a/backend/plugin/parser/spanner/split.go+++ b/backend/plugin/parser/spanner/split.go@@ -57,6 +57,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {return result, nil}+ byteOffset := 0start := 0for i, stmt := range allStatements {// Find the semicolon after this statement@@ -88,13 +89,16 @@ func SplitSQL(statement string) ([]base.Statement, error) {continue}+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[endPos])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : endPos+1])result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[endPos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[endPos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{Line: int32(tokens[endPos].GetLine()),@@ -103,6 +107,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:endPos+1], parser.GoogleSQLLexerSEMI_SYMBOL),})+ byteOffset += stmtByteLengthstart = endPos + 1}diff --git a/backend/plugin/parser/tidb/split.go b/backend/plugin/parser/tidb/split.goindex a940a93a5f..61195ca126 100644--- a/backend/plugin/parser/tidb/split.go+++ b/backend/plugin/parser/tidb/split.go@@ -72,6 +72,7 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([delimiter := ";"tokens := stream.GetAllTokens()start := 0+ byteOffset := 0i := 0for i < len(tokens) {@@ -91,15 +92,18 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([// Deal with normal statement.if delimiter == ";" && token.GetTokenType() == parser.MySQLLexerSEMICOLON_SYMBOL {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[i])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : i+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[i]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[i].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -111,6 +115,7 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:i+1], parser.MySQLLexerSEMICOLON_SYMBOL),})+ byteOffset += stmtByteLengthi++start = icontinue@@ -122,6 +127,9 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([}if newStart, ok := tryMatchDelimiter(stream, i, delimiter); ok {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[newStart-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:newStart])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.@@ -130,8 +138,8 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([Text: stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";",BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[newStart-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -143,6 +151,7 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:i], parser.MySQLLexerSEMICOLON_SYMBOL),})+ byteOffset += stmtByteLengthi = newStartstart = newStartcontinue@@ -153,15 +162,18 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([endPos := len(tokens) - 1if start < endPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[endPos-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[endPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[endPos-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -319,17 +331,21 @@ func splitTiDBStatement(stream *antlr.CommonTokenStream, statement string) ([]ba}}+ byteOffset := 0start := 0for _, pos := range semicolonStack {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -341,20 +357,24 @@ func splitTiDBStatement(stream *antlr.CommonTokenStream, statement string) ([]baStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),})+ byteOffset += stmtByteLengthstart = pos + 1}// For the last statement, it may not end with semicolon symbol, EOF symbol instead.eofPos := len(tokens) - 1if start < eofPos {+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])+ stmtByteLength := len(stmtText)+antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])// From antlr4, the line is ONE based, and the column is ZERO based.// So we should minus 1 for the line.sresult = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[eofPos-1].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{diff --git a/backend/plugin/parser/trino/split.go b/backend/plugin/parser/trino/split.goindex 3727ef128a..5d8e9beb4c 100644--- a/backend/plugin/parser/trino/split.go+++ b/backend/plugin/parser/trino/split.go@@ -133,6 +133,7 @@ func splitByParser(statement string) ([]base.Statement, error) {var result []base.Statementtokens := stream.GetAllTokens()+ byteOffset := 0// Walk through all statementsfor _, stmts := range tree.AllStatements() {if stmts == nil {@@ -182,6 +183,7 @@ func splitByParser(statement string) ([]base.Statement, error) {// Get the text including any trailing semicolontext := stream.GetTextFromInterval(antlr.NewInterval(startIdx, finalEndIdx))+ stmtByteLength := len(text)// Calculate proper end positionendToken := tokens[finalEndIdx]@@ -192,8 +194,8 @@ func splitByParser(statement string) ([]base.Statement, error) {Text: text,BaseLine: firstToken.GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[startIdx].GetStart()),- End: int32(endToken.GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},Start: &storepb.Position{Line: int32(firstDefaultToken.GetLine() - 1),@@ -205,6 +207,7 @@ func splitByParser(statement string) ([]base.Statement, error) {},Empty: false,})+ byteOffset += stmtByteLength}return result, nildiff --git a/backend/plugin/parser/trino/split_test.go b/backend/plugin/parser/trino/split_test.goindex 26984263e5..54662800b8 100644--- a/backend/plugin/parser/trino/split_test.go+++ b/backend/plugin/parser/trino/split_test.go@@ -109,6 +109,25 @@ func TestTrinoSplitMultiSQL(t *testing.T) {},},},+ {+ statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",+ want: resData{+ res: []base.Statement{+ {+ Text: "SELECT * FROM 表名;",+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)+ Start: &storepb.Position{Line: 0, Column: 0},+ End: &storepb.Position{Line: 0, Column: 17},+ },+ {+ Text: "INSERT INTO 表 VALUES (1);",+ Range: &storepb.Range{Start: 21, End: 48}, // Byte offset 21-48 (not 17-42)+ Start: &storepb.Position{Line: 0, Column: 18},+ End: &storepb.Position{Line: 0, Column: 42},+ },+ },+ },+ },}for _, test := range tests {diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.goindex d8f8daf6c4..794cc0c178 100644--- a/backend/plugin/parser/tsql/split.go+++ b/backend/plugin/parser/tsql/split.go@@ -67,18 +67,21 @@ func splitByParser(statement string) ([]base.Statement, error) {var result []base.Statementtokens := stream.GetAllTokens()start := 0+ byteOffset := 0if len(tree.AllBatch_without_go()) == 0 {// Go statement only.for _, goStmt := range tree.AllGo_statement() {pos := goStmt.GetStop().GetTokenIndex()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -90,6 +93,7 @@ func splitByParser(statement string) ([]base.Statement, error) {Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: false,})+ byteOffset += stmtByteLengthstart = pos + 1}return result, nil@@ -98,7 +102,7 @@ func splitByParser(statement string) ([]base.Statement, error) {// First batch_without_go.b := tree.AllBatch_without_go()[0]var r []base.Statement- r, start = splitBatchWithoutGo(b, tokens, stream, start, statement)+ r, start, byteOffset = splitBatchWithoutGo(b, tokens, stream, start, byteOffset, statement)result = append(result, r...)goIdx := 0@@ -118,13 +122,15 @@ func splitByParser(statement string) ([]base.Statement, error) {for _, goStmt := range goStmts {pos := goStmt.GetStop().GetTokenIndex()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -136,10 +142,11 @@ func splitByParser(statement string) ([]base.Statement, error) {Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: false,})+ byteOffset += stmtByteLengthstart = pos + 1}- r, start = splitBatchWithoutGo(b, tokens, stream, start, statement)+ r, start, byteOffset = splitBatchWithoutGo(b, tokens, stream, start, byteOffset, statement)result = append(result, r...)}}@@ -148,13 +155,15 @@ func splitByParser(statement string) ([]base.Statement, error) {// Last go statement.for _, goStmt := range tree.AllGo_statement()[goIdx:] {pos := goStmt.GetStop().GetTokenIndex()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -166,6 +175,7 @@ func splitByParser(statement string) ([]base.Statement, error) {Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: false,})+ byteOffset += stmtByteLengthstart = pos + 1}}@@ -173,20 +183,22 @@ func splitByParser(statement string) ([]base.Statement, error) {return result, nil}-func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token, stream *antlr.CommonTokenStream, start int, statement string) ([]base.Statement, int) {+func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token, stream *antlr.CommonTokenStream, start int, byteOffset int, statement string) ([]base.Statement, int, int) {var result []base.Statementswitch {case b.Batch_level_statement() == nil && b.Execute_body_batch() == nil:// All sql_clauses.for _, sqlClause := range b.AllSql_clauses() {pos := sqlClause.GetStop().GetTokenIndex()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -198,17 +210,20 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.TokenStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: false,})+ byteOffset += stmtByteLengthstart = pos + 1}case b.Batch_level_statement() != nil:pos := b.Batch_level_statement().GetStop().GetTokenIndex()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -220,16 +235,19 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.TokenStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: false,})+ byteOffset += stmtByteLengthstart = pos + 1case b.Execute_body_batch() != nil:pos := b.Execute_body_batch().GetStop().GetTokenIndex()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -241,16 +259,19 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.TokenStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: false,})+ byteOffset += stmtByteLengthstart = pos + 1for _, sqlClause := range b.AllSql_clauses() {pos := sqlClause.GetStop().GetTokenIndex()+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])+ stmtByteLength := len(stmtText)antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])result = append(result, base.Statement{- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),+ Text: stmtText,BaseLine: tokens[start].GetLine() - 1,Range: &storepb.Range{- Start: int32(tokens[start].GetStart()),- End: int32(tokens[pos].GetStop() + 1),+ Start: int32(byteOffset),+ End: int32(byteOffset + stmtByteLength),},End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{@@ -262,10 +283,11 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.TokenStart: common.ConvertANTLRPositionToPosition(antlrPosition, statement),Empty: false,})+ byteOffset += stmtByteLengthstart = pos + 1}default:// No statements found in this batch}- return result, start+ return result, start, byteOffset}diff --git a/backend/plugin/parser/tsql/split_test.go b/backend/plugin/parser/tsql/split_test.goindex 38b218d925..36ed7ada4b 100644--- a/backend/plugin/parser/tsql/split_test.go+++ b/backend/plugin/parser/tsql/split_test.go@@ -129,6 +129,29 @@ UPDATE SalesLT.ProductModelProductDescription SET Culture = "zh-cn";},},},+ {+ statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",+ want: resData{+ res: []base.Statement{+ {+ Text: "SELECT * FROM 表名;",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 1},+ End: &storepb.Position{Line: 1, Column: 17},+ Empty: false,+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)+ },+ {+ Text: " INSERT INTO 表 VALUES (1);",+ BaseLine: 0,+ Start: &storepb.Position{Line: 1, Column: 19},+ End: &storepb.Position{Line: 1, Column: 43},+ Empty: false,+ Range: &storepb.Range{Start: 21, End: 49}, // Byte offset 21-49 (not 17-43)+ },+ },+ },+ },}for _, test := range tests {diff --git a/proto/store/store/common.proto b/proto/store/store/common.protoindex 6e03338d5f..02d8c43334 100644--- a/proto/store/store/common.proto+++ b/proto/store/store/common.proto@@ -108,6 +108,8 @@ message Position {}// Range represents a span within a text or sequence.+// Whether the indices are byte offsets or character indices depends on the context.+// Check the documentation of the field using Range for specific semantics.message Range {// Start index (inclusive).int32 start = 1;diff --git a/proto/store/store/setting.proto b/proto/store/store/setting.protoindex 3190370f41..dfeb964de9 100644--- a/proto/store/store/setting.proto+++ b/proto/store/store/setting.proto@@ -189,11 +189,14 @@ message Algorithm {message RangeMask {message Slice {- // start is the start index of the original value, start from 0 and should be less than stop.+ // start is the start character index (0-based) of the original value, should be less than end.+ // Uses character indices (not byte offsets) for display-oriented masking.+ // Example: For "你好world", character index 2 refers to 'w' (the 3rd character).int32 start = 1;- // stop is the stop index of the original value, should be less than the length of the original value.+ // end is the end character index (exclusive) of the original value.+ // Uses character indices (not byte offsets) for display-oriented masking.int32 end = 2;- // OriginalValue[start:end) would be replaced with replace_with.+ // OriginalValue[start:end) would be replaced with substitution.string substitution = 3;}// We store it as a repeated field to face the fact that the original value may have multiple parts should be masked.diff --git a/proto/store/store/task_run_log.proto b/proto/store/store/task_run_log.protoindex f9e475ddc0..ebfaff71fc 100644--- a/proto/store/store/task_run_log.proto+++ b/proto/store/store/task_run_log.proto@@ -46,6 +46,8 @@ message TaskRunLog {}message CommandExecute {// The byte offset range of the executed command in the sheet.+ // Uses byte offsets (not character indices) for efficient slicing of sheet content bytes.+ // Example: For "SELECT 你好;" in a UTF-8 sheet, range [0, 13) represents all 13 bytes.Range range = 1;// The statement to be executed.diff --git a/proto/v1/v1/common.proto b/proto/v1/v1/common.protoindex 1d48834eaf..9660eb9bb2 100644--- a/proto/v1/v1/common.proto+++ b/proto/v1/v1/common.proto@@ -127,11 +127,13 @@ message Position {int32 column = 2;}-// Range of positions in text or sequence.+// Range represents a span within a text or sequence.+// Whether the indices are byte offsets or character indices depends on the context.+// Check the documentation of the field using Range for specific semantics.message Range {- // Start position (inclusive).+ // Start index (inclusive).int32 start = 1;- // End position (exclusive).+ // End index (exclusive).int32 end = 2;}diff --git a/proto/v1/v1/rollout_service.proto b/proto/v1/v1/rollout_service.protoindex 4952b5dd25..220ee93830 100644--- a/proto/v1/v1/rollout_service.proto+++ b/proto/v1/v1/rollout_service.proto@@ -672,6 +672,8 @@ message TaskRunLogEntry {google.protobuf.Timestamp log_time = 1;// The byte offset range of the executed command in the sheet.+ // Uses byte offsets (not character indices) for efficient slicing of sheet content bytes.+ // Example: For "SELECT 你好;" in a UTF-8 sheet, range [0, 13) represents all 13 bytes.Range range = 2;// The executed statement.diff --git a/proto/v1/v1/setting_service.proto b/proto/v1/v1/setting_service.protoindex 47ff7c39d3..ee80a3ca63 100644--- a/proto/v1/v1/setting_service.proto+++ b/proto/v1/v1/setting_service.proto@@ -362,9 +362,12 @@ message Algorithm {message RangeMask {message Slice {- // start is the start index of the original value, start from 0 and should be less than stop.+ // start is the start character index (0-based) of the original value, should be less than end.+ // Uses character indices (not byte offsets) for display-oriented masking.+ // Example: For "你好world", character index 2 refers to 'w' (the 3rd character).int32 start = 1;- // stop is the stop index of the original value, should be less than the length of the original value.+ // end is the end character index (exclusive) of the original value.+ // Uses character indices (not byte offsets) for display-oriented masking.int32 end = 2;// substitution is the string used to replace the OriginalValue[start:end).string substitution = 3;