Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat: use byte offsets for statement ranges to support UTF-8 correctly (#18536)

Danny Xubf7e827b2cb3
diff --git a/backend/plugin/parser/bigquery/split.go b/backend/plugin/parser/bigquery/split.go
index b6da131f29..71aa0e828d 100644
--- a/backend/plugin/parser/bigquery/split.go
+++ b/backend/plugin/parser/bigquery/split.go
@@ -23,6 +23,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {
tokens := stream.GetAllTokens()
var buf []antlr.Token
var sqls []base.Statement
+ byteOffset := 0
for i, token := range tokens {
if i < len(tokens)-1 {
buf = append(buf, token)
@@ -39,13 +40,15 @@ func SplitSQL(statement string) ([]base.Statement, error) {
empty = false
}
}
+ stmtText := bufStr.String()
+ stmtByteLength := len(stmtText)
antlrPosition := base.FirstDefaultChannelTokenPosition(buf)
sqls = append(sqls, base.Statement{
- Text: bufStr.String(),
+ Text: stmtText,
BaseLine: buf[0].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(buf[0].GetStart()),
- End: int32(buf[len(buf)-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(buf[len(buf)-1].GetLine()),
@@ -54,6 +57,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: empty,
})
+ byteOffset += stmtByteLength
buf = nil
continue
}
diff --git a/backend/plugin/parser/mysql/split.go b/backend/plugin/parser/mysql/split.go
index eebf948c90..e8b86781de 100644
--- a/backend/plugin/parser/mysql/split.go
+++ b/backend/plugin/parser/mysql/split.go
@@ -56,15 +56,25 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
// Deal with normal statement.
if delimiter == ";" && token.GetTokenType() == parser.MySQLLexerSEMICOLON_SYMBOL {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[i])
+
+ // Calculate byte range by getting actual bytes from original statement
+ // This works because for ASCII chars, char offset == byte offset
+ // For UTF-8, we get the actual substring which handles multi-byte chars correctly
+ stmtStartChar := tokens[start].GetStart()
+ stmtEndChar := tokens[i].GetStop() + 1
+ stmtStartByte := len(statement[:stmtStartChar])
+ stmtEndByte := len(statement[:stmtEndChar])
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : i+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[i]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[i].GetStop() + 1),
+ Start: int32(stmtStartByte),
+ End: int32(stmtEndByte),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[i].GetLine()),
@@ -84,16 +94,24 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
}
if newStart, ok := tryMatchDelimiter(stream, i, delimiter); ok {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";"
+
+ // Calculate byte range by getting actual bytes from original statement
+ stmtStartChar := tokens[start].GetStart()
+ stmtEndChar := tokens[newStart-1].GetStop() + 1
+ stmtStartByte := len(statement[:stmtStartChar])
+ stmtEndByte := len(statement[:stmtEndChar])
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:newStart])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
// Use a single semicolon instead of the user defined delimiter.
- Text: stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";",
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[newStart-1].GetStop() + 1),
+ Start: int32(stmtStartByte),
+ End: int32(stmtEndByte),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[newStart-1].GetLine()),
@@ -112,15 +130,23 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
endPos := len(tokens) - 1
if start < endPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[endPos-1])
+
+ // Calculate byte range by getting actual bytes from original statement
+ stmtStartChar := tokens[start].GetStart()
+ stmtEndChar := tokens[endPos-1].GetStop() + 1
+ stmtStartByte := len(statement[:stmtStartChar])
+ stmtEndByte := len(statement[:stmtEndChar])
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[endPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[endPos-1].GetStop() + 1),
+ Start: int32(stmtStartByte),
+ End: int32(stmtEndByte),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[endPos-1].GetLine()),
@@ -205,18 +231,22 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Com
var result []base.Statement
tokens := stream.GetAllTokens()
+ byteOffset := 0
start := 0
for _, semicolon := range tree.AllSEMICOLON_SYMBOL() {
pos := semicolon.GetSymbol().GetStart()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -225,20 +255,24 @@ func splitByParser(statement string, lexer *parser.MySQLLexer, stream *antlr.Com
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),
})
+ byteOffset += stmtByteLength
start = pos + 1
}
// For the last statement, it may not end with semicolon symbol, EOF symbol instead.
eofPos := len(tokens) - 1
if start < eofPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[eofPos-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
@@ -392,17 +426,21 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b
}
}
+ byteOffset := 0
start := 0
for _, pos := range semicolonStack {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -411,20 +449,24 @@ func splitMySQLStatement(stream *antlr.CommonTokenStream, statement string) ([]b
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),
})
+ byteOffset += stmtByteLength
start = pos + 1
}
// For the last statement, it may not end with semicolon symbol, EOF symbol instead.
eofPos := len(tokens) - 1
if start < eofPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[eofPos-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
diff --git a/backend/plugin/parser/mysql/split_test.go b/backend/plugin/parser/mysql/split_test.go
index 67e132b159..afc9356988 100644
--- a/backend/plugin/parser/mysql/split_test.go
+++ b/backend/plugin/parser/mysql/split_test.go
@@ -643,6 +643,25 @@ func TestMySQLSplitMultiSQL(t *testing.T) {
},
},
},
+ {
+ statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",
+ want: resData{
+ res: []base.Statement{
+ {
+ Text: "SELECT * FROM 表名;",
+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 17},
+ },
+ {
+ Text: " INSERT INTO 表 VALUES (1);",
+ Range: &storepb.Range{Start: 21, End: 49}, // Byte offset 21-49 (not 17-43)
+ Start: &storepb.Position{Line: 1, Column: 19},
+ End: &storepb.Position{Line: 1, Column: 43},
+ },
+ },
+ },
+ },
}
for _, test := range tests {
diff --git a/backend/plugin/parser/pg/split.go b/backend/plugin/parser/pg/split.go
index 113a465fda..23ac44ca5a 100644
--- a/backend/plugin/parser/pg/split.go
+++ b/backend/plugin/parser/pg/split.go
@@ -63,18 +63,22 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antl
var result []base.Statement
tokens := stream.GetAllTokens()
+ byteOffset := 0
start := 0
for _, semi := range tree.Stmtblock().Stmtmulti().AllSEMI() {
pos := semi.GetSymbol().GetStart()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -83,20 +87,24 @@ func splitByParser(statement string, lexer *parser.PostgreSQLLexer, stream *antl
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),
})
+ byteOffset += stmtByteLength
start = pos + 1
}
// For the last statement, it may not end with semicolon symbol, EOF symbol instead.
eofPos := len(tokens) - 1
if start < eofPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[eofPos-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
@@ -191,17 +199,21 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
}
}
+ byteOffset := 0
start := 0
for _, pos := range semicolonStack {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -210,20 +222,24 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:pos+1], parser.PostgreSQLParserSEMI),
})
+ byteOffset += stmtByteLength
start = pos + 1
}
// For the last statement, it may not end with semicolon symbol, EOF symbol instead.
eofPos := len(tokens) - 1
if start < eofPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[eofPos-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
diff --git a/backend/plugin/parser/pg/split_test.go b/backend/plugin/parser/pg/split_test.go
index 86307aee98..a9a1480cf6 100644
--- a/backend/plugin/parser/pg/split_test.go
+++ b/backend/plugin/parser/pg/split_test.go
@@ -341,6 +341,25 @@ func TestPGSplitMultiSQL(t *testing.T) {
},
},
},
+ {
+ statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",
+ want: resData{
+ res: []base.Statement{
+ {
+ Text: "SELECT * FROM 表名;",
+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 17},
+ },
+ {
+ Text: " INSERT INTO 表 VALUES (1);",
+ Range: &storepb.Range{Start: 21, End: 49}, // Byte offset 21-49 (not 17-43)
+ Start: &storepb.Position{Line: 1, Column: 19},
+ End: &storepb.Position{Line: 1, Column: 43},
+ },
+ },
+ },
+ },
}
for _, test := range tests {
diff --git a/backend/plugin/parser/redshift/split.go b/backend/plugin/parser/redshift/split.go
index 521d6e7043..cb4172db5c 100644
--- a/backend/plugin/parser/redshift/split.go
+++ b/backend/plugin/parser/redshift/split.go
@@ -64,18 +64,22 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.
var result []base.Statement
tokens := stream.GetAllTokens()
+ byteOffset := 0
start := 0
for _, semi := range tree.Stmtblock().Stmtmulti().AllSEMI() {
pos := semi.GetSymbol().GetStart()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -84,20 +88,24 @@ func splitByParser(statement string, lexer *parser.RedshiftLexer, stream *antlr.
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:pos+1], parser.RedshiftParserSEMI),
})
+ byteOffset += stmtByteLength
start = pos + 1
}
// For the last statement, it may not end with semicolon symbol, EOF symbol instead.
eofPos := len(tokens) - 1
if start < eofPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[eofPos-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
@@ -191,17 +199,21 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
}
}
+ byteOffset := 0
start := 0
for _, pos := range semicolonStack {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[pos].GetLine()),
@@ -210,20 +222,24 @@ func splitSQLImpl(stream *antlr.CommonTokenStream, statement string) ([]base.Sta
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:pos+1], parser.RedshiftParserSEMI),
})
+ byteOffset += stmtByteLength
start = pos + 1
}
// For the last statement, it may not end with semicolon symbol, EOF symbol instead.
eofPos := len(tokens) - 1
if start < eofPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[eofPos-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[eofPos-1].GetLine()),
diff --git a/backend/plugin/parser/spanner/split.go b/backend/plugin/parser/spanner/split.go
index 4068b0964f..df592d009b 100644
--- a/backend/plugin/parser/spanner/split.go
+++ b/backend/plugin/parser/spanner/split.go
@@ -57,6 +57,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {
return result, nil
}
+ byteOffset := 0
start := 0
for i, stmt := range allStatements {
// Find the semicolon after this statement
@@ -88,13 +89,16 @@ func SplitSQL(statement string) ([]base.Statement, error) {
continue
}
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[endPos])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : endPos+1])
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[endPos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[endPos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(&common.ANTLRPosition{
Line: int32(tokens[endPos].GetLine()),
@@ -103,6 +107,7 @@ func SplitSQL(statement string) ([]base.Statement, error) {
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:endPos+1], parser.GoogleSQLLexerSEMI_SYMBOL),
})
+ byteOffset += stmtByteLength
start = endPos + 1
}
diff --git a/backend/plugin/parser/tidb/split.go b/backend/plugin/parser/tidb/split.go
index a940a93a5f..61195ca126 100644
--- a/backend/plugin/parser/tidb/split.go
+++ b/backend/plugin/parser/tidb/split.go
@@ -72,6 +72,7 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
delimiter := ";"
tokens := stream.GetAllTokens()
start := 0
+ byteOffset := 0
i := 0
for i < len(tokens) {
@@ -91,15 +92,18 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
// Deal with normal statement.
if delimiter == ";" && token.GetTokenType() == parser.MySQLLexerSEMICOLON_SYMBOL {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[i])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : i+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[i]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[i].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -111,6 +115,7 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:i+1], parser.MySQLLexerSEMICOLON_SYMBOL),
})
+ byteOffset += stmtByteLength
i++
start = i
continue
@@ -122,6 +127,9 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
}
if newStart, ok := tryMatchDelimiter(stream, i, delimiter); ok {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[newStart-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:newStart])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
@@ -130,8 +138,8 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
Text: stream.GetTextFromTokens(tokens[start], tokens[i-1]) + ";",
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[newStart-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -143,6 +151,7 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:i], parser.MySQLLexerSEMICOLON_SYMBOL),
})
+ byteOffset += stmtByteLength
i = newStart
start = newStart
continue
@@ -153,15 +162,18 @@ func splitDelimiterModeSQL(stream *antlr.CommonTokenStream, statement string) ([
endPos := len(tokens) - 1
if start < endPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[endPos-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[endPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[endPos-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -319,17 +331,21 @@ func splitTiDBStatement(stream *antlr.CommonTokenStream, statement string) ([]ba
}
}
+ byteOffset := 0
start := 0
for _, pos := range semicolonStack {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -341,20 +357,24 @@ func splitTiDBStatement(stream *antlr.CommonTokenStream, statement string) ([]ba
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: base.IsEmpty(tokens[start:pos+1], parser.MySQLLexerSEMICOLON_SYMBOL),
})
+ byteOffset += stmtByteLength
start = pos + 1
}
// For the last statement, it may not end with semicolon symbol, EOF symbol instead.
eofPos := len(tokens) - 1
if start < eofPos {
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[eofPos-1])
+ stmtByteLength := len(stmtText)
+
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start:])
// From antlr4, the line is ONE based, and the column is ZERO based.
// So we should minus 1 for the line.s
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[eofPos-1]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[eofPos-1].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
diff --git a/backend/plugin/parser/trino/split.go b/backend/plugin/parser/trino/split.go
index 3727ef128a..5d8e9beb4c 100644
--- a/backend/plugin/parser/trino/split.go
+++ b/backend/plugin/parser/trino/split.go
@@ -133,6 +133,7 @@ func splitByParser(statement string) ([]base.Statement, error) {
var result []base.Statement
tokens := stream.GetAllTokens()
+ byteOffset := 0
// Walk through all statements
for _, stmts := range tree.AllStatements() {
if stmts == nil {
@@ -182,6 +183,7 @@ func splitByParser(statement string) ([]base.Statement, error) {
// Get the text including any trailing semicolon
text := stream.GetTextFromInterval(antlr.NewInterval(startIdx, finalEndIdx))
+ stmtByteLength := len(text)
// Calculate proper end position
endToken := tokens[finalEndIdx]
@@ -192,8 +194,8 @@ func splitByParser(statement string) ([]base.Statement, error) {
Text: text,
BaseLine: firstToken.GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[startIdx].GetStart()),
- End: int32(endToken.GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
Start: &storepb.Position{
Line: int32(firstDefaultToken.GetLine() - 1),
@@ -205,6 +207,7 @@ func splitByParser(statement string) ([]base.Statement, error) {
},
Empty: false,
})
+ byteOffset += stmtByteLength
}
return result, nil
diff --git a/backend/plugin/parser/trino/split_test.go b/backend/plugin/parser/trino/split_test.go
index 26984263e5..54662800b8 100644
--- a/backend/plugin/parser/trino/split_test.go
+++ b/backend/plugin/parser/trino/split_test.go
@@ -109,6 +109,25 @@ func TestTrinoSplitMultiSQL(t *testing.T) {
},
},
},
+ {
+ statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",
+ want: resData{
+ res: []base.Statement{
+ {
+ Text: "SELECT * FROM 表名;",
+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)
+ Start: &storepb.Position{Line: 0, Column: 0},
+ End: &storepb.Position{Line: 0, Column: 17},
+ },
+ {
+ Text: "INSERT INTO 表 VALUES (1);",
+ Range: &storepb.Range{Start: 21, End: 48}, // Byte offset 21-48 (not 17-42)
+ Start: &storepb.Position{Line: 0, Column: 18},
+ End: &storepb.Position{Line: 0, Column: 42},
+ },
+ },
+ },
+ },
}
for _, test := range tests {
diff --git a/backend/plugin/parser/tsql/split.go b/backend/plugin/parser/tsql/split.go
index d8f8daf6c4..794cc0c178 100644
--- a/backend/plugin/parser/tsql/split.go
+++ b/backend/plugin/parser/tsql/split.go
@@ -67,18 +67,21 @@ func splitByParser(statement string) ([]base.Statement, error) {
var result []base.Statement
tokens := stream.GetAllTokens()
start := 0
+ byteOffset := 0
if len(tree.AllBatch_without_go()) == 0 {
// Go statement only.
for _, goStmt := range tree.AllGo_statement() {
pos := goStmt.GetStop().GetTokenIndex()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -90,6 +93,7 @@ func splitByParser(statement string) ([]base.Statement, error) {
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: false,
})
+ byteOffset += stmtByteLength
start = pos + 1
}
return result, nil
@@ -98,7 +102,7 @@ func splitByParser(statement string) ([]base.Statement, error) {
// First batch_without_go.
b := tree.AllBatch_without_go()[0]
var r []base.Statement
- r, start = splitBatchWithoutGo(b, tokens, stream, start, statement)
+ r, start, byteOffset = splitBatchWithoutGo(b, tokens, stream, start, byteOffset, statement)
result = append(result, r...)
goIdx := 0
@@ -118,13 +122,15 @@ func splitByParser(statement string) ([]base.Statement, error) {
for _, goStmt := range goStmts {
pos := goStmt.GetStop().GetTokenIndex()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -136,10 +142,11 @@ func splitByParser(statement string) ([]base.Statement, error) {
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: false,
})
+ byteOffset += stmtByteLength
start = pos + 1
}
- r, start = splitBatchWithoutGo(b, tokens, stream, start, statement)
+ r, start, byteOffset = splitBatchWithoutGo(b, tokens, stream, start, byteOffset, statement)
result = append(result, r...)
}
}
@@ -148,13 +155,15 @@ func splitByParser(statement string) ([]base.Statement, error) {
// Last go statement.
for _, goStmt := range tree.AllGo_statement()[goIdx:] {
pos := goStmt.GetStop().GetTokenIndex()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -166,6 +175,7 @@ func splitByParser(statement string) ([]base.Statement, error) {
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: false,
})
+ byteOffset += stmtByteLength
start = pos + 1
}
}
@@ -173,20 +183,22 @@ func splitByParser(statement string) ([]base.Statement, error) {
return result, nil
}
-func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token, stream *antlr.CommonTokenStream, start int, statement string) ([]base.Statement, int) {
+func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token, stream *antlr.CommonTokenStream, start int, byteOffset int, statement string) ([]base.Statement, int, int) {
var result []base.Statement
switch {
case b.Batch_level_statement() == nil && b.Execute_body_batch() == nil:
// All sql_clauses.
for _, sqlClause := range b.AllSql_clauses() {
pos := sqlClause.GetStop().GetTokenIndex()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -198,17 +210,20 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: false,
})
+ byteOffset += stmtByteLength
start = pos + 1
}
case b.Batch_level_statement() != nil:
pos := b.Batch_level_statement().GetStop().GetTokenIndex()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -220,16 +235,19 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: false,
})
+ byteOffset += stmtByteLength
start = pos + 1
case b.Execute_body_batch() != nil:
pos := b.Execute_body_batch().GetStop().GetTokenIndex()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -241,16 +259,19 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: false,
})
+ byteOffset += stmtByteLength
start = pos + 1
for _, sqlClause := range b.AllSql_clauses() {
pos := sqlClause.GetStop().GetTokenIndex()
+ stmtText := stream.GetTextFromTokens(tokens[start], tokens[pos])
+ stmtByteLength := len(stmtText)
antlrPosition := base.FirstDefaultChannelTokenPosition(tokens[start : pos+1])
result = append(result, base.Statement{
- Text: stream.GetTextFromTokens(tokens[start], tokens[pos]),
+ Text: stmtText,
BaseLine: tokens[start].GetLine() - 1,
Range: &storepb.Range{
- Start: int32(tokens[start].GetStart()),
- End: int32(tokens[pos].GetStop() + 1),
+ Start: int32(byteOffset),
+ End: int32(byteOffset + stmtByteLength),
},
End: common.ConvertANTLRPositionToPosition(
&common.ANTLRPosition{
@@ -262,10 +283,11 @@ func splitBatchWithoutGo(b parser.IBatch_without_goContext, tokens []antlr.Token
Start: common.ConvertANTLRPositionToPosition(antlrPosition, statement),
Empty: false,
})
+ byteOffset += stmtByteLength
start = pos + 1
}
default:
// No statements found in this batch
}
- return result, start
+ return result, start, byteOffset
}
diff --git a/backend/plugin/parser/tsql/split_test.go b/backend/plugin/parser/tsql/split_test.go
index 38b218d925..36ed7ada4b 100644
--- a/backend/plugin/parser/tsql/split_test.go
+++ b/backend/plugin/parser/tsql/split_test.go
@@ -129,6 +129,29 @@ UPDATE SalesLT.ProductModelProductDescription SET Culture = "zh-cn";
},
},
},
+ {
+ statement: "SELECT * FROM 表名; INSERT INTO 表 VALUES (1);",
+ want: resData{
+ res: []base.Statement{
+ {
+ Text: "SELECT * FROM 表名;",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 1},
+ End: &storepb.Position{Line: 1, Column: 17},
+ Empty: false,
+ Range: &storepb.Range{Start: 0, End: 21}, // Byte offset 0-21 (not 0-17)
+ },
+ {
+ Text: " INSERT INTO 表 VALUES (1);",
+ BaseLine: 0,
+ Start: &storepb.Position{Line: 1, Column: 19},
+ End: &storepb.Position{Line: 1, Column: 43},
+ Empty: false,
+ Range: &storepb.Range{Start: 21, End: 49}, // Byte offset 21-49 (not 17-43)
+ },
+ },
+ },
+ },
}
for _, test := range tests {
diff --git a/proto/store/store/common.proto b/proto/store/store/common.proto
index 6e03338d5f..02d8c43334 100644
--- a/proto/store/store/common.proto
+++ b/proto/store/store/common.proto
@@ -108,6 +108,8 @@ message Position {
}
// Range represents a span within a text or sequence.
+// Whether the indices are byte offsets or character indices depends on the context.
+// Check the documentation of the field using Range for specific semantics.
message Range {
// Start index (inclusive).
int32 start = 1;
diff --git a/proto/store/store/setting.proto b/proto/store/store/setting.proto
index 3190370f41..dfeb964de9 100644
--- a/proto/store/store/setting.proto
+++ b/proto/store/store/setting.proto
@@ -189,11 +189,14 @@ message Algorithm {
message RangeMask {
message Slice {
- // start is the start index of the original value, start from 0 and should be less than stop.
+ // start is the start character index (0-based) of the original value, should be less than end.
+ // Uses character indices (not byte offsets) for display-oriented masking.
+ // Example: For "你好world", character index 2 refers to 'w' (the 3rd character).
int32 start = 1;
- // stop is the stop index of the original value, should be less than the length of the original value.
+ // end is the end character index (exclusive) of the original value.
+ // Uses character indices (not byte offsets) for display-oriented masking.
int32 end = 2;
- // OriginalValue[start:end) would be replaced with replace_with.
+ // OriginalValue[start:end) would be replaced with substitution.
string substitution = 3;
}
// We store it as a repeated field to face the fact that the original value may have multiple parts should be masked.
diff --git a/proto/store/store/task_run_log.proto b/proto/store/store/task_run_log.proto
index f9e475ddc0..ebfaff71fc 100644
--- a/proto/store/store/task_run_log.proto
+++ b/proto/store/store/task_run_log.proto
@@ -46,6 +46,8 @@ message TaskRunLog {
}
message CommandExecute {
// The byte offset range of the executed command in the sheet.
+ // Uses byte offsets (not character indices) for efficient slicing of sheet content bytes.
+ // Example: For "SELECT 你好;" in a UTF-8 sheet, range [0, 13) represents all 13 bytes.
Range range = 1;
// The statement to be executed.
diff --git a/proto/v1/v1/common.proto b/proto/v1/v1/common.proto
index 1d48834eaf..9660eb9bb2 100644
--- a/proto/v1/v1/common.proto
+++ b/proto/v1/v1/common.proto
@@ -127,11 +127,13 @@ message Position {
int32 column = 2;
}
-// Range of positions in text or sequence.
+// Range represents a span within a text or sequence.
+// Whether the indices are byte offsets or character indices depends on the context.
+// Check the documentation of the field using Range for specific semantics.
message Range {
- // Start position (inclusive).
+ // Start index (inclusive).
int32 start = 1;
- // End position (exclusive).
+ // End index (exclusive).
int32 end = 2;
}
diff --git a/proto/v1/v1/rollout_service.proto b/proto/v1/v1/rollout_service.proto
index 4952b5dd25..220ee93830 100644
--- a/proto/v1/v1/rollout_service.proto
+++ b/proto/v1/v1/rollout_service.proto
@@ -672,6 +672,8 @@ message TaskRunLogEntry {
google.protobuf.Timestamp log_time = 1;
// The byte offset range of the executed command in the sheet.
+ // Uses byte offsets (not character indices) for efficient slicing of sheet content bytes.
+ // Example: For "SELECT 你好;" in a UTF-8 sheet, range [0, 13) represents all 13 bytes.
Range range = 2;
// The executed statement.
diff --git a/proto/v1/v1/setting_service.proto b/proto/v1/v1/setting_service.proto
index 47ff7c39d3..ee80a3ca63 100644
--- a/proto/v1/v1/setting_service.proto
+++ b/proto/v1/v1/setting_service.proto
@@ -362,9 +362,12 @@ message Algorithm {
message RangeMask {
message Slice {
- // start is the start index of the original value, start from 0 and should be less than stop.
+ // start is the start character index (0-based) of the original value, should be less than end.
+ // Uses character indices (not byte offsets) for display-oriented masking.
+ // Example: For "你好world", character index 2 refers to 'w' (the 3rd character).
int32 start = 1;
- // stop is the stop index of the original value, should be less than the length of the original value.
+ // end is the end character index (exclusive) of the original value.
+ // Uses character indices (not byte offsets) for display-oriented masking.
int32 end = 2;
// substitution is the string used to replace the OriginalValue[start:end).
string substitution = 3;