Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(parser): fix CRLF line endings causing query truncation in SQL splitter (#19821)
Danny Xu5d92e98140ca
diff --git a/backend/plugin/parser/base/lexer.go b/backend/plugin/parser/base/lexer.goindex 74e56b0a50..3aaba8660a 100644--- a/backend/plugin/parser/base/lexer.go+++ b/backend/plugin/parser/base/lexer.go@@ -144,12 +144,25 @@ func CalculateLineAndColumn(statement string, byteOffset int) (line, column int)if byteOffset > len(statement) {byteOffset = len(statement)}- // Range over string iterates over runes (code points), not bytes- for _, r := range statement[:byteOffset] {- if r == '\n' {+ // Range over string iterates over runes (code points), not bytes.+ // \r\n is treated as a single line break; standalone \r is a line break.+ s := statement[:byteOffset]+ for i, r := range s {+ switch r {+ case '\r':+ // \r\n is one line break; standalone \r is also a line break.+ // In both cases, increment line here. For \r\n, the \n case+ // below will see that the previous char was \r and skip.line++column = 0- } else {+ case '\n':+ if i > 0 && s[i-1] == '\r' {+ // Already counted by the \r above.+ continue+ }+ line+++ column = 0+ default:column++}}diff --git a/backend/plugin/parser/standard/split.go b/backend/plugin/parser/standard/split.goindex 3d848b229f..83f152af6e 100644--- a/backend/plugin/parser/standard/split.go+++ b/backend/plugin/parser/standard/split.go@@ -22,30 +22,9 @@ func init() {// SplitSQL splits the given SQL statement into multiple SQL statements.func SplitSQL(statement string) ([]base.Statement, error) {var list []base.Statement- byteOffset := 0- err := applyMultiStatements(strings.NewReader(statement), func(sql string) error {- // sql may have trailing newline added by applyMultiStatements- // Strip trailing whitespace for searching in the original statement- sqlTrimmed := strings.TrimRight(sql, "\n\r\t ")-- // Find where the SQL content exists in the remaining statement- sqlPos := strings.Index(statement[byteOffset:], sqlTrimmed)- if sqlPos == -1 {- sqlPos = 0- }-- // startPos includes leading whitespace from where previous statement ended- startPos := byteOffset- // endPos is where the SQL content ends- endPos := byteOffset + sqlPos + len(sqlTrimmed)-- // The actual text includes leading whitespace + SQL content- text := statement[startPos:endPos]-- // Calculate line and column for Start position- startLine, startColumn := base.CalculateLineAndColumn(statement, startPos)- // Calculate line and column for End position- endLine, endColumn := base.CalculateLineAndColumn(statement, endPos)+ err := applyMultiStatements(statement, func(text string, start, end int) error {+ startLine, startColumn := base.CalculateLineAndColumn(statement, start)+ endLine, endColumn := base.CalculateLineAndColumn(statement, end)list = append(list, base.Statement{Text: text,@@ -58,12 +37,11 @@ func SplitSQL(statement string) ([]base.Statement, error) {Column: int32(endColumn + 1), // 1-based per proto spec},Range: &storepb.Range{- Start: int32(startPos),- End: int32(endPos),+ Start: int32(start),+ End: int32(end),},Empty: isEmptySQL(text),})- byteOffset = endPosreturn nil})return list, err@@ -98,26 +76,34 @@ func isEmptySQL(sql string) bool {return false}-// applyMultiStatements will apply the split statements from scanner.-// This function only used for SQLite, snowflake.-// For MySQL and PostgreSQL, use parser.SplitSQL.-// Copy from plugin/db/util/driverutil.go.-func applyMultiStatements(sc io.Reader, f func(string) error) error {- // TODO(rebelice): use parser/tokenizer to split SQL statements.- reader := bufio.NewReader(sc)- var sb strings.Builder+// applyMultiStatements splits the statement by semicolons and invokes f for+// each sub-statement with the text slice from the original statement and its+// [start, end) byte offsets. The text for each statement includes any leading+// whitespace from the end of the previous statement, matching the original+// behavior needed for position tracking.+func applyMultiStatements(statement string, f func(text string, start, end int) error) error {+ reader := bufio.NewReader(strings.NewReader(statement))delimiter := falsecomment := falsedone := false+ hasContent := false+ // byteOffset tracks our read position in the original statement.+ byteOffset := 0+ // prevEnd tracks where the last emitted statement ended; the next+ // statement's text will start here (to include inter-statement whitespace).+ prevEnd := 0+ // contentEnd is the byte offset just past the last content character of+ // the statement being accumulated (excludes trailing whitespace/newlines).+ contentEnd := 0for !done {line, err := reader.ReadString('\n')if err != nil {- if err == io.EOF {- done = true- } else {+ if err != io.EOF {return err}+ done = true}+ lineLen := len(line)line = strings.TrimRightFunc(line, unicode.IsSpace)execute := false@@ -130,56 +116,50 @@ func applyMultiStatements(sc io.Reader, f func(string) error) error {} else {comment = true}- continuecase comment && !strings.Contains(line, "*/"):- // Skip the line when in comment mode.- continue+ // skip line in comment modecase comment && strings.Contains(line, "*/"):if !strings.HasSuffix(line, "*/") {return errors.Errorf("`*/` must be the end of the line; new statement should start as a new line")}comment = false- continue- case sb.Len() == 0 && line == "":- continue+ case !hasContent && line == "":+ // skip leading blank linescase strings.HasPrefix(line, "--"):- continue+ // skip comment linescase line == "DELIMITER ;;":delimiter = true- continuecase line == "DELIMITER ;" && delimiter:delimiter = falseexecute = truecase strings.HasSuffix(line, ";"):- _, _ = sb.WriteString(line)- _, _ = sb.WriteString("\n")+ hasContent = true+ contentEnd = byteOffset + len(line)if !delimiter {execute = true}default:- _, _ = sb.WriteString(line)- _, _ = sb.WriteString("\n")- continue+ if line != "" {+ hasContent = true+ contentEnd = byteOffset + len(line)+ }}- if execute {- s := sb.String()- // Don't trim - include leading whitespace in Text for position consistency- if s != "" {- if err := f(s); err != nil {- return errors.Wrapf(err, "execute query %q failed", s)- }+ if execute && hasContent {+ text := statement[prevEnd:contentEnd]+ if err := f(text, prevEnd, contentEnd); err != nil {+ return errors.Wrapf(err, "execute query %q failed", text)}- sb.Reset()+ hasContent = false+ prevEnd = contentEnd}+ byteOffset += lineLen}// Apply the remaining content.- s := sb.String()- // Don't trim - include leading whitespace in Text for position consistency- if s != "" {- if err := f(s); err != nil {- return errors.Wrapf(err, "execute query %q failed", s)+ if hasContent {+ text := statement[prevEnd:contentEnd]+ if err := f(text, prevEnd, contentEnd); err != nil {+ return errors.Wrapf(err, "execute query %q failed", text)}}-return nil}diff --git a/backend/plugin/parser/standard/split_test.go b/backend/plugin/parser/standard/split_test.goindex 7564e7c65d..073dde1d26 100644--- a/backend/plugin/parser/standard/split_test.go+++ b/backend/plugin/parser/standard/split_test.go@@ -4,7 +4,6 @@ import ("crypto/rand""fmt""math/big"- "strings""testing""github.com/stretchr/testify/require"@@ -69,14 +68,14 @@ func TestApplyMultiStatements(t *testing.T) {}total := 0- countStatements := func(string) error {+ countStatements := func(_ string, _, _ int) error {total++return nil}for _, test := range tests {total = 0- err := applyMultiStatements(strings.NewReader(test.statement), countStatements)+ err := applyMultiStatements(test.statement, countStatements)require.NoError(t, err)require.Equal(t, test.total, total)}diff --git a/backend/plugin/parser/standard/test-data/test_split.yaml b/backend/plugin/parser/standard/test-data/test_split.yamlindex ed4a61600b..c578170879 100644--- a/backend/plugin/parser/standard/test-data/test_split.yaml+++ b/backend/plugin/parser/standard/test-data/test_split.yaml@@ -184,3 +184,60 @@start: 9end: 22empty: false+- description: 'CRLF: multi-line without semicolon'+ input: "select col_a,count(col_b)\r\nfrom tbl\r\ngroup by col_a"+ result:+ - text: "select col_a,count(col_b)\r\nfrom tbl\r\ngroup by col_a"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 3+ column: 15+ range:+ start: 0+ end: 51+ empty: false+- description: 'CRLF: multiple statements'+ input: "SELECT 1;\r\nSELECT 2;"+ result:+ - text: "SELECT 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 1+ column: 10+ range:+ start: 0+ end: 9+ empty: false+ - text: "\r\nSELECT 2;"+ baseline: 0+ start:+ line: 1+ column: 10+ end:+ line: 2+ column: 10+ range:+ start: 9+ end: 20+ empty: false+- description: 'CRLF: single statement with semicolon'+ input: "SELECT\r\n 1;"+ result:+ - text: "SELECT\r\n 1;"+ baseline: 0+ start:+ line: 1+ column: 1+ end:+ line: 2+ column: 5+ range:+ start: 0+ end: 12+ empty: false