Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(tidb): correct line tracking for blank-line-separated statements (BYT-9381) (#20474)

Vincent Huang2457110e92bf
diff --git a/backend/plugin/parser/tidb/omni_test.go b/backend/plugin/parser/tidb/omni_test.go
index a74a13f12d..f40b39dad4 100644
--- a/backend/plugin/parser/tidb/omni_test.go
+++ b/backend/plugin/parser/tidb/omni_test.go
@@ -3,6 +3,7 @@ package tidb
import (
"testing"
+ "github.com/pingcap/tidb/pkg/parser/ast"
"github.com/stretchr/testify/require"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
@@ -280,3 +281,99 @@ func TestAsPingCapASTLineTrackingMatchesCanonical(t *testing.T) {
a.Equal(2, bridged.Node.OriginTextPosition(),
"sanity: ALTER TABLE on line 2 of the input should report OriginTextPosition=2")
}
+
+// TestAsPingCapASTLineTrackingMatchesCanonicalBlankLines exercises the
+// leadingNewlinesStripped arithmetic in applyTiDBLineTracking (BYT-9381). With
+// blank lines between statements, the native pingcap parser strips the leading
+// newlines from the second statement's Text(), so the bridge must add them back
+// to report the correct absolute OriginTextPosition. The single-newline sibling
+// test above keeps leadingNewlinesStripped == 0 and so never covers this path;
+// a regression in the arithmetic would otherwise land silently.
+func TestAsPingCapASTLineTrackingMatchesCanonicalBlankLines(t *testing.T) {
+ a := require.New(t)
+
+ // Two blank lines between the statements: ALTER TABLE is on line 4.
+ multi := "CREATE TABLE foo (id INT);\n\n\nALTER TABLE foo ADD COLUMN x INT NOT NULL;"
+
+ canonical, err := ParseTiDBForSyntaxCheck(multi)
+ a.NoError(err)
+ a.Len(canonical, 2)
+ canonicalSecond, ok := canonical[1].(*AST)
+ a.True(ok)
+
+ splits, err := base.SplitMultiSQL(storepb.Engine_TIDB, multi)
+ a.NoError(err)
+ a.GreaterOrEqual(len(splits), 2)
+ secondSplit := splits[1]
+
+ o := &OmniAST{
+ Text: secondSplit.Text,
+ StartPosition: &storepb.Position{Line: int32(secondSplit.BaseLine()) + 1},
+ }
+ bridged, ok := o.AsPingCapAST()
+ a.True(ok)
+
+ a.Equal(
+ canonicalSecond.Node.OriginTextPosition(),
+ bridged.Node.OriginTextPosition(),
+ "bridge OriginTextPosition must match ParseTiDBForSyntaxCheck across blank lines",
+ )
+ a.Equal(4, bridged.Node.OriginTextPosition(),
+ "sanity: ALTER TABLE on line 4 (after two blank lines) should report OriginTextPosition=4")
+}
+
+// TestApplyTiDBLineTracking pins the leading-newline arithmetic directly across
+// the edges the integration tests don't all exercise: no leading whitespace
+// (line 1), single vs multiple leading newlines, CRLF line endings, a non-zero
+// baseLine, and non-newline leading whitespace (BYT-9381).
+func TestApplyTiDBLineTracking(t *testing.T) {
+ nodes, err := ParseTiDB("SELECT 1", "", "")
+ require.NoError(t, err)
+ require.Len(t, nodes, 1)
+ node := nodes[0]
+
+ cases := []struct {
+ name string
+ baseLine int
+ text string
+ want int
+ }{
+ {"no leading whitespace is line 1", 0, "SELECT 1", 1},
+ {"single leading newline", 0, "\nSELECT 1", 2},
+ {"three leading newlines (the BYT-9381 bug)", 0, "\n\n\nSELECT 1", 4},
+ {"CRLF leading newlines count once each", 0, "\r\n\r\n\r\nSELECT 1", 4},
+ {"non-zero baseLine plus two newlines", 5, "\n\nSELECT 1", 8},
+ {"leading spaces and tabs, no newline", 0, " \tSELECT 1", 1},
+ {"mixed leading whitespace with one newline", 0, " \n SELECT 1", 2},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ got, err := applyTiDBLineTracking(node, tc.baseLine, tc.text)
+ require.NoError(t, err)
+ require.Equal(t, tc.want, got)
+ })
+ }
+}
+
+// TestCreateTableColumnLineTrackingBlankLines pins per-column line tracking for
+// a CREATE TABLE preceded by blank lines (BYT-9381). The native parser may
+// retain leading newlines in node.Text(); the column tokenizer counts from the
+// start of that text, so its base line must exclude those leading newlines —
+// otherwise the blank lines are double-counted into every column's line (the
+// statement line is correct but columns land several lines too low).
+func TestCreateTableColumnLineTrackingBlankLines(t *testing.T) {
+ a := require.New(t)
+
+ // CREATE TABLE on line 4 (after two blank lines); id on line 5, name on 6.
+ multi := "SELECT 1;\n\n\nCREATE TABLE foo (\n id INT,\n name VARCHAR(50)\n);"
+ asts, err := ParseTiDBForSyntaxCheck(multi)
+ a.NoError(err)
+ a.Len(asts, 2)
+ ct, ok := asts[1].(*AST).Node.(*ast.CreateTableStmt)
+ a.True(ok)
+
+ a.Equal(4, ct.OriginTextPosition(), "CREATE TABLE statement line")
+ a.Len(ct.Cols, 2)
+ a.Equal(5, ct.Cols[0].OriginTextPosition(), "column id should be on line 5")
+ a.Equal(6, ct.Cols[1].OriginTextPosition(), "column name should be on line 6")
+}
diff --git a/backend/plugin/parser/tidb/tidb.go b/backend/plugin/parser/tidb/tidb.go
index 2ef430b5d9..58477d0980 100644
--- a/backend/plugin/parser/tidb/tidb.go
+++ b/backend/plugin/parser/tidb/tidb.go
@@ -71,11 +71,15 @@ func ParseTiDBForSyntaxCheck(statement string) ([]base.AST, error) {
// Returns the 1-based actualStartLine (`baseLine + leadingNewlinesStripped +
// 1`) so callers can use it for *AST.StartPosition.
func applyTiDBLineTracking(node ast.StmtNode, baseLine int, originalText string) (int, error) {
- // The native TiDB parser may strip leading newlines from its internal
- // Text(); count how many to recover the absolute line.
- nativeText := node.Text()
- leadingNewlinesStripped := strings.Count(originalText, "\n") - strings.Count(nativeText, "\n")
- actualStartLine := baseLine + leadingNewlinesStripped + 1
+ // The statement's absolute start line is baseLine plus the number of
+ // newlines in its leading whitespace (the blank lines before the first
+ // token). Counting the leading newlines of originalText directly is robust;
+ // the native parser's node.Text() does not reliably strip ALL leading
+ // newlines, so diffing original-vs-native newline counts under-counts when
+ // more than one blank line precedes the statement (BYT-9381). This keeps the
+ // pingcap-bridge line in step with the omni path's FirstTokenLine.
+ leadingWhitespace := originalText[:len(originalText)-len(strings.TrimLeft(originalText, " \t\r\n"))]
+ actualStartLine := baseLine + strings.Count(leadingWhitespace, "\n") + 1
node.SetOriginTextPosition(actualStartLine)
if n, ok := node.(*ast.CreateTableStmt); ok {
@@ -153,10 +157,15 @@ func SetLineForMySQLCreateTableStmt(node *ast.CreateTableStmt) error {
if len(node.Cols) == 0 {
return nil
}
- // OriginTextPosition() now stores the first line of the statement (1-based),
- // so we can use it directly as firstLine.
- firstLine := node.OriginTextPosition()
- return tokenizer.NewTokenizer(node.Text()).SetLineForMySQLCreateTableStmt(node, firstLine)
+ // node.OriginTextPosition() is the statement's first line (the CREATE
+ // keyword). The tokenizer counts newlines from the START of node.Text(),
+ // which may retain leading newlines the native parser didn't strip, so its
+ // base must be the line of node.Text()'s first character — the statement
+ // line minus those leading newlines. Otherwise blank lines before the
+ // statement are double-counted into every column/constraint line (BYT-9381).
+ text := node.Text()
+ leadingNewlines := strings.Count(text[:len(text)-len(strings.TrimLeft(text, " \t\r\n"))], "\n")
+ return tokenizer.NewTokenizer(text).SetLineForMySQLCreateTableStmt(node, node.OriginTextPosition()-leadingNewlines)
}
// TypeString returns the string representation of the type for MySQL.
diff --git a/backend/plugin/parser/tidb/tidb_test.go b/backend/plugin/parser/tidb/tidb_test.go
index 78f39ef5bc..581d20caf6 100644
--- a/backend/plugin/parser/tidb/tidb_test.go
+++ b/backend/plugin/parser/tidb/tidb_test.go
@@ -71,7 +71,11 @@ func TestMySQLCreateTableSetLine(t *testing.T) {
"\r\n" +
"FOREIGN KEY (a, b, c) REFERENCES t1(a, b, c)" + "\r\n" +
")",
- firstLine: 1,
+ // Leading \r\n puts the first token (CREATE) on line 2; firstLine is
+ // the first-non-whitespace line per the post-BYT-9381 contract for
+ // node.OriginTextPosition() (SetLineForMySQLCreateTableStmt converts it
+ // back to node.Text()'s char-0 line for the tokenizer).
+ firstLine: 2,
columnLineList: []int{3, 3, 4, 5},
constraintLineList: []int{6, 7, 8, 8, 10},
},