Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(parser): fix CRLF line endings causing query truncation in SQL splitter (#19821)

Danny Xu5d92e98140ca
diff --git a/backend/plugin/parser/base/lexer.go b/backend/plugin/parser/base/lexer.go
index 74e56b0a50..3aaba8660a 100644
--- a/backend/plugin/parser/base/lexer.go
+++ b/backend/plugin/parser/base/lexer.go
@@ -144,12 +144,25 @@ func CalculateLineAndColumn(statement string, byteOffset int) (line, column int)
if byteOffset > len(statement) {
byteOffset = len(statement)
}
- // Range over string iterates over runes (code points), not bytes
- for _, r := range statement[:byteOffset] {
- if r == '\n' {
+ // Range over string iterates over runes (code points), not bytes.
+ // \r\n is treated as a single line break; standalone \r is a line break.
+ s := statement[:byteOffset]
+ for i, r := range s {
+ switch r {
+ case '\r':
+ // \r\n is one line break; standalone \r is also a line break.
+ // In both cases, increment line here. For \r\n, the \n case
+ // below will see that the previous char was \r and skip.
line++
column = 0
- } else {
+ case '\n':
+ if i > 0 && s[i-1] == '\r' {
+ // Already counted by the \r above.
+ continue
+ }
+ line++
+ column = 0
+ default:
column++
}
}
diff --git a/backend/plugin/parser/standard/split.go b/backend/plugin/parser/standard/split.go
index 3d848b229f..83f152af6e 100644
--- a/backend/plugin/parser/standard/split.go
+++ b/backend/plugin/parser/standard/split.go
@@ -22,30 +22,9 @@ func init() {
// SplitSQL splits the given SQL statement into multiple SQL statements.
func SplitSQL(statement string) ([]base.Statement, error) {
var list []base.Statement
- byteOffset := 0
- err := applyMultiStatements(strings.NewReader(statement), func(sql string) error {
- // sql may have trailing newline added by applyMultiStatements
- // Strip trailing whitespace for searching in the original statement
- sqlTrimmed := strings.TrimRight(sql, "\n\r\t ")
-
- // Find where the SQL content exists in the remaining statement
- sqlPos := strings.Index(statement[byteOffset:], sqlTrimmed)
- if sqlPos == -1 {
- sqlPos = 0
- }
-
- // startPos includes leading whitespace from where previous statement ended
- startPos := byteOffset
- // endPos is where the SQL content ends
- endPos := byteOffset + sqlPos + len(sqlTrimmed)
-
- // The actual text includes leading whitespace + SQL content
- text := statement[startPos:endPos]
-
- // Calculate line and column for Start position
- startLine, startColumn := base.CalculateLineAndColumn(statement, startPos)
- // Calculate line and column for End position
- endLine, endColumn := base.CalculateLineAndColumn(statement, endPos)
+ err := applyMultiStatements(statement, func(text string, start, end int) error {
+ startLine, startColumn := base.CalculateLineAndColumn(statement, start)
+ endLine, endColumn := base.CalculateLineAndColumn(statement, end)
list = append(list, base.Statement{
Text: text,
@@ -58,12 +37,11 @@ func SplitSQL(statement string) ([]base.Statement, error) {
Column: int32(endColumn + 1), // 1-based per proto spec
},
Range: &storepb.Range{
- Start: int32(startPos),
- End: int32(endPos),
+ Start: int32(start),
+ End: int32(end),
},
Empty: isEmptySQL(text),
})
- byteOffset = endPos
return nil
})
return list, err
@@ -98,26 +76,34 @@ func isEmptySQL(sql string) bool {
return false
}
-// applyMultiStatements will apply the split statements from scanner.
-// This function only used for SQLite, snowflake.
-// For MySQL and PostgreSQL, use parser.SplitSQL.
-// Copy from plugin/db/util/driverutil.go.
-func applyMultiStatements(sc io.Reader, f func(string) error) error {
- // TODO(rebelice): use parser/tokenizer to split SQL statements.
- reader := bufio.NewReader(sc)
- var sb strings.Builder
+// applyMultiStatements splits the statement by semicolons and invokes f for
+// each sub-statement with the text slice from the original statement and its
+// [start, end) byte offsets. The text for each statement includes any leading
+// whitespace from the end of the previous statement, matching the original
+// behavior needed for position tracking.
+func applyMultiStatements(statement string, f func(text string, start, end int) error) error {
+ reader := bufio.NewReader(strings.NewReader(statement))
delimiter := false
comment := false
done := false
+ hasContent := false
+ // byteOffset tracks our read position in the original statement.
+ byteOffset := 0
+ // prevEnd tracks where the last emitted statement ended; the next
+ // statement's text will start here (to include inter-statement whitespace).
+ prevEnd := 0
+ // contentEnd is the byte offset just past the last content character of
+ // the statement being accumulated (excludes trailing whitespace/newlines).
+ contentEnd := 0
for !done {
line, err := reader.ReadString('\n')
if err != nil {
- if err == io.EOF {
- done = true
- } else {
+ if err != io.EOF {
return err
}
+ done = true
}
+ lineLen := len(line)
line = strings.TrimRightFunc(line, unicode.IsSpace)
execute := false
@@ -130,56 +116,50 @@ func applyMultiStatements(sc io.Reader, f func(string) error) error {
} else {
comment = true
}
- continue
case comment && !strings.Contains(line, "*/"):
- // Skip the line when in comment mode.
- continue
+ // skip line in comment mode
case comment && strings.Contains(line, "*/"):
if !strings.HasSuffix(line, "*/") {
return errors.Errorf("`*/` must be the end of the line; new statement should start as a new line")
}
comment = false
- continue
- case sb.Len() == 0 && line == "":
- continue
+ case !hasContent && line == "":
+ // skip leading blank lines
case strings.HasPrefix(line, "--"):
- continue
+ // skip comment lines
case line == "DELIMITER ;;":
delimiter = true
- continue
case line == "DELIMITER ;" && delimiter:
delimiter = false
execute = true
case strings.HasSuffix(line, ";"):
- _, _ = sb.WriteString(line)
- _, _ = sb.WriteString("\n")
+ hasContent = true
+ contentEnd = byteOffset + len(line)
if !delimiter {
execute = true
}
default:
- _, _ = sb.WriteString(line)
- _, _ = sb.WriteString("\n")
- continue
+ if line != "" {
+ hasContent = true
+ contentEnd = byteOffset + len(line)
+ }
}
- if execute {
- s := sb.String()
- // Don't trim - include leading whitespace in Text for position consistency
- if s != "" {
- if err := f(s); err != nil {
- return errors.Wrapf(err, "execute query %q failed", s)
- }
+ if execute && hasContent {
+ text := statement[prevEnd:contentEnd]
+ if err := f(text, prevEnd, contentEnd); err != nil {
+ return errors.Wrapf(err, "execute query %q failed", text)
}
- sb.Reset()
+ hasContent = false
+ prevEnd = contentEnd
}
+ byteOffset += lineLen
}
// Apply the remaining content.
- s := sb.String()
- // Don't trim - include leading whitespace in Text for position consistency
- if s != "" {
- if err := f(s); err != nil {
- return errors.Wrapf(err, "execute query %q failed", s)
+ if hasContent {
+ text := statement[prevEnd:contentEnd]
+ if err := f(text, prevEnd, contentEnd); err != nil {
+ return errors.Wrapf(err, "execute query %q failed", text)
}
}
-
return nil
}
diff --git a/backend/plugin/parser/standard/split_test.go b/backend/plugin/parser/standard/split_test.go
index 7564e7c65d..073dde1d26 100644
--- a/backend/plugin/parser/standard/split_test.go
+++ b/backend/plugin/parser/standard/split_test.go
@@ -4,7 +4,6 @@ import (
"crypto/rand"
"fmt"
"math/big"
- "strings"
"testing"
"github.com/stretchr/testify/require"
@@ -69,14 +68,14 @@ func TestApplyMultiStatements(t *testing.T) {
}
total := 0
- countStatements := func(string) error {
+ countStatements := func(_ string, _, _ int) error {
total++
return nil
}
for _, test := range tests {
total = 0
- err := applyMultiStatements(strings.NewReader(test.statement), countStatements)
+ err := applyMultiStatements(test.statement, countStatements)
require.NoError(t, err)
require.Equal(t, test.total, total)
}
diff --git a/backend/plugin/parser/standard/test-data/test_split.yaml b/backend/plugin/parser/standard/test-data/test_split.yaml
index ed4a61600b..c578170879 100644
--- a/backend/plugin/parser/standard/test-data/test_split.yaml
+++ b/backend/plugin/parser/standard/test-data/test_split.yaml
@@ -184,3 +184,60 @@
start: 9
end: 22
empty: false
+- description: 'CRLF: multi-line without semicolon'
+ input: "select col_a,count(col_b)\r\nfrom tbl\r\ngroup by col_a"
+ result:
+ - text: "select col_a,count(col_b)\r\nfrom tbl\r\ngroup by col_a"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 3
+ column: 15
+ range:
+ start: 0
+ end: 51
+ empty: false
+- description: 'CRLF: multiple statements'
+ input: "SELECT 1;\r\nSELECT 2;"
+ result:
+ - text: "SELECT 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 1
+ column: 10
+ range:
+ start: 0
+ end: 9
+ empty: false
+ - text: "\r\nSELECT 2;"
+ baseline: 0
+ start:
+ line: 1
+ column: 10
+ end:
+ line: 2
+ column: 10
+ range:
+ start: 9
+ end: 20
+ empty: false
+- description: 'CRLF: single statement with semicolon'
+ input: "SELECT\r\n 1;"
+ result:
+ - text: "SELECT\r\n 1;"
+ baseline: 0
+ start:
+ line: 1
+ column: 1
+ end:
+ line: 2
+ column: 5
+ range:
+ start: 0
+ end: 12
+ empty: false