Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(tidb): correct line tracking for blank-line-separated statements (BYT-9381) (#20474)
Vincent Huang2457110e92bf
diff --git a/backend/plugin/parser/tidb/omni_test.go b/backend/plugin/parser/tidb/omni_test.goindex a74a13f12d..f40b39dad4 100644--- a/backend/plugin/parser/tidb/omni_test.go+++ b/backend/plugin/parser/tidb/omni_test.go@@ -3,6 +3,7 @@ package tidbimport ("testing"+ "github.com/pingcap/tidb/pkg/parser/ast""github.com/stretchr/testify/require"storepb "github.com/bytebase/bytebase/backend/generated-go/store"@@ -280,3 +281,99 @@ func TestAsPingCapASTLineTrackingMatchesCanonical(t *testing.T) {a.Equal(2, bridged.Node.OriginTextPosition(),"sanity: ALTER TABLE on line 2 of the input should report OriginTextPosition=2")}++// TestAsPingCapASTLineTrackingMatchesCanonicalBlankLines exercises the+// leadingNewlinesStripped arithmetic in applyTiDBLineTracking (BYT-9381). With+// blank lines between statements, the native pingcap parser strips the leading+// newlines from the second statement's Text(), so the bridge must add them back+// to report the correct absolute OriginTextPosition. The single-newline sibling+// test above keeps leadingNewlinesStripped == 0 and so never covers this path;+// a regression in the arithmetic would otherwise land silently.+func TestAsPingCapASTLineTrackingMatchesCanonicalBlankLines(t *testing.T) {+ a := require.New(t)++ // Two blank lines between the statements: ALTER TABLE is on line 4.+ multi := "CREATE TABLE foo (id INT);\n\n\nALTER TABLE foo ADD COLUMN x INT NOT NULL;"++ canonical, err := ParseTiDBForSyntaxCheck(multi)+ a.NoError(err)+ a.Len(canonical, 2)+ canonicalSecond, ok := canonical[1].(*AST)+ a.True(ok)++ splits, err := base.SplitMultiSQL(storepb.Engine_TIDB, multi)+ a.NoError(err)+ a.GreaterOrEqual(len(splits), 2)+ secondSplit := splits[1]++ o := &OmniAST{+ Text: secondSplit.Text,+ StartPosition: &storepb.Position{Line: int32(secondSplit.BaseLine()) + 1},+ }+ bridged, ok := o.AsPingCapAST()+ a.True(ok)++ a.Equal(+ canonicalSecond.Node.OriginTextPosition(),+ bridged.Node.OriginTextPosition(),+ "bridge OriginTextPosition must match ParseTiDBForSyntaxCheck across blank lines",+ )+ a.Equal(4, bridged.Node.OriginTextPosition(),+ "sanity: ALTER TABLE on line 4 (after two blank lines) should report OriginTextPosition=4")+}++// TestApplyTiDBLineTracking pins the leading-newline arithmetic directly across+// the edges the integration tests don't all exercise: no leading whitespace+// (line 1), single vs multiple leading newlines, CRLF line endings, a non-zero+// baseLine, and non-newline leading whitespace (BYT-9381).+func TestApplyTiDBLineTracking(t *testing.T) {+ nodes, err := ParseTiDB("SELECT 1", "", "")+ require.NoError(t, err)+ require.Len(t, nodes, 1)+ node := nodes[0]++ cases := []struct {+ name string+ baseLine int+ text string+ want int+ }{+ {"no leading whitespace is line 1", 0, "SELECT 1", 1},+ {"single leading newline", 0, "\nSELECT 1", 2},+ {"three leading newlines (the BYT-9381 bug)", 0, "\n\n\nSELECT 1", 4},+ {"CRLF leading newlines count once each", 0, "\r\n\r\n\r\nSELECT 1", 4},+ {"non-zero baseLine plus two newlines", 5, "\n\nSELECT 1", 8},+ {"leading spaces and tabs, no newline", 0, " \tSELECT 1", 1},+ {"mixed leading whitespace with one newline", 0, " \n SELECT 1", 2},+ }+ for _, tc := range cases {+ t.Run(tc.name, func(t *testing.T) {+ got, err := applyTiDBLineTracking(node, tc.baseLine, tc.text)+ require.NoError(t, err)+ require.Equal(t, tc.want, got)+ })+ }+}++// TestCreateTableColumnLineTrackingBlankLines pins per-column line tracking for+// a CREATE TABLE preceded by blank lines (BYT-9381). The native parser may+// retain leading newlines in node.Text(); the column tokenizer counts from the+// start of that text, so its base line must exclude those leading newlines —+// otherwise the blank lines are double-counted into every column's line (the+// statement line is correct but columns land several lines too low).+func TestCreateTableColumnLineTrackingBlankLines(t *testing.T) {+ a := require.New(t)++ // CREATE TABLE on line 4 (after two blank lines); id on line 5, name on 6.+ multi := "SELECT 1;\n\n\nCREATE TABLE foo (\n id INT,\n name VARCHAR(50)\n);"+ asts, err := ParseTiDBForSyntaxCheck(multi)+ a.NoError(err)+ a.Len(asts, 2)+ ct, ok := asts[1].(*AST).Node.(*ast.CreateTableStmt)+ a.True(ok)++ a.Equal(4, ct.OriginTextPosition(), "CREATE TABLE statement line")+ a.Len(ct.Cols, 2)+ a.Equal(5, ct.Cols[0].OriginTextPosition(), "column id should be on line 5")+ a.Equal(6, ct.Cols[1].OriginTextPosition(), "column name should be on line 6")+}diff --git a/backend/plugin/parser/tidb/tidb.go b/backend/plugin/parser/tidb/tidb.goindex 2ef430b5d9..58477d0980 100644--- a/backend/plugin/parser/tidb/tidb.go+++ b/backend/plugin/parser/tidb/tidb.go@@ -71,11 +71,15 @@ func ParseTiDBForSyntaxCheck(statement string) ([]base.AST, error) {// Returns the 1-based actualStartLine (`baseLine + leadingNewlinesStripped +// 1`) so callers can use it for *AST.StartPosition.func applyTiDBLineTracking(node ast.StmtNode, baseLine int, originalText string) (int, error) {- // The native TiDB parser may strip leading newlines from its internal- // Text(); count how many to recover the absolute line.- nativeText := node.Text()- leadingNewlinesStripped := strings.Count(originalText, "\n") - strings.Count(nativeText, "\n")- actualStartLine := baseLine + leadingNewlinesStripped + 1+ // The statement's absolute start line is baseLine plus the number of+ // newlines in its leading whitespace (the blank lines before the first+ // token). Counting the leading newlines of originalText directly is robust;+ // the native parser's node.Text() does not reliably strip ALL leading+ // newlines, so diffing original-vs-native newline counts under-counts when+ // more than one blank line precedes the statement (BYT-9381). This keeps the+ // pingcap-bridge line in step with the omni path's FirstTokenLine.+ leadingWhitespace := originalText[:len(originalText)-len(strings.TrimLeft(originalText, " \t\r\n"))]+ actualStartLine := baseLine + strings.Count(leadingWhitespace, "\n") + 1node.SetOriginTextPosition(actualStartLine)if n, ok := node.(*ast.CreateTableStmt); ok {@@ -153,10 +157,15 @@ func SetLineForMySQLCreateTableStmt(node *ast.CreateTableStmt) error {if len(node.Cols) == 0 {return nil}- // OriginTextPosition() now stores the first line of the statement (1-based),- // so we can use it directly as firstLine.- firstLine := node.OriginTextPosition()- return tokenizer.NewTokenizer(node.Text()).SetLineForMySQLCreateTableStmt(node, firstLine)+ // node.OriginTextPosition() is the statement's first line (the CREATE+ // keyword). The tokenizer counts newlines from the START of node.Text(),+ // which may retain leading newlines the native parser didn't strip, so its+ // base must be the line of node.Text()'s first character — the statement+ // line minus those leading newlines. Otherwise blank lines before the+ // statement are double-counted into every column/constraint line (BYT-9381).+ text := node.Text()+ leadingNewlines := strings.Count(text[:len(text)-len(strings.TrimLeft(text, " \t\r\n"))], "\n")+ return tokenizer.NewTokenizer(text).SetLineForMySQLCreateTableStmt(node, node.OriginTextPosition()-leadingNewlines)}// TypeString returns the string representation of the type for MySQL.diff --git a/backend/plugin/parser/tidb/tidb_test.go b/backend/plugin/parser/tidb/tidb_test.goindex 78f39ef5bc..581d20caf6 100644--- a/backend/plugin/parser/tidb/tidb_test.go+++ b/backend/plugin/parser/tidb/tidb_test.go@@ -71,7 +71,11 @@ func TestMySQLCreateTableSetLine(t *testing.T) {"\r\n" +"FOREIGN KEY (a, b, c) REFERENCES t1(a, b, c)" + "\r\n" +")",- firstLine: 1,+ // Leading \r\n puts the first token (CREATE) on line 2; firstLine is+ // the first-non-whitespace line per the post-BYT-9381 contract for+ // node.OriginTextPosition() (SetLineForMySQLCreateTableStmt converts it+ // back to node.Text()'s char-0 line for the tokenizer).+ firstLine: 2,columnLineList: []int{3, 3, 4, 5},constraintLineList: []int{6, 7, 8, 8, 10},},