Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(mysql,tidb): always split statements instead of merging large sheets
Vincent Huang395e5d075d7c
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.goindex f34cc07bed..fc524b8e49 100644--- a/backend/plugin/db/mysql/mysql.go+++ b/backend/plugin/db/mysql/mysql.go@@ -307,11 +307,11 @@ func parseVersion(version string) (string, string, error) {return "", "", errors.Errorf("failed to parse version %q", version)}+// buildExecuteCommands splits the statement into single commands.+// Commands are never merged back into one: a merged sheet is sent as a single+// COM_QUERY whose size equals the sheet size, tripping max_allowed_packet on+// large DML, and it collapses per-statement execution logging.func buildExecuteCommands(statement string) ([]base.Statement, error) {- if len(statement) > common.MaxSheetCheckSize {- return []base.Statement{{Text: statement}}, nil- }-commands, err := mysqlparser.SplitSQL(statement)if err != nil {return nil, errors.Wrapf(err, "failed to split sql")@@ -320,9 +320,6 @@ func buildExecuteCommands(statement string) ([]base.Statement, error) {if len(commands) == 0 {return nil, nil}- if len(commands) > common.MaximumCommands {- return []base.Statement{{Text: statement}}, nil- }return commands, nil}diff --git a/backend/plugin/db/mysql/mysql_test.go b/backend/plugin/db/mysql/mysql_test.goindex 32d326bdf5..4b38d14d9a 100644--- a/backend/plugin/db/mysql/mysql_test.go+++ b/backend/plugin/db/mysql/mysql_test.go@@ -1,6 +1,7 @@package mysqlimport (+ "fmt""strings""testing"@@ -127,7 +128,7 @@ func TestBuildExecuteCommandsNormalizesDelimiter(t *testing.T) {require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")}-func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForTooManyCommands(t *testing.T) {+func TestBuildExecuteCommandsNormalizesDelimiterForManyCommands(t *testing.T) {var statement strings.Builderstatement.WriteString("DELIMITER //\n")statement.WriteString("CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND//\n")@@ -139,16 +140,39 @@ func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForTooManyCommands(t *testcommands, err := buildExecuteCommands(statement.String())require.NoError(t, err)- require.Len(t, commands, 1)- require.Equal(t, statement.String(), commands[0].Text)+ require.Greater(t, len(commands), common.MaximumCommands)+ require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")+ for _, command := range commands {+ require.NotContains(t, command.Text, "DELIMITER")+ }}-func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForLargeSheet(t *testing.T) {+func TestBuildExecuteCommandsNormalizesDelimiterForLargeSheet(t *testing.T) {statement := "DELIMITER //\nCREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND//\nDELIMITER ;\n" +strings.Repeat(" ", common.MaxSheetCheckSize)commands, err := buildExecuteCommands(statement)require.NoError(t, err)require.Len(t, commands, 1)- require.Equal(t, statement, commands[0].Text)+ require.NotContains(t, commands[0].Text, "DELIMITER")+ require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")+}++// A sheet over MaxSheetCheckSize used to be sent as one multi-statement packet,+// which trips MySQL's max_allowed_packet with error 1153 on large DML backfills.+func TestBuildExecuteCommandsSplitsOversizedSheet(t *testing.T) {+ const statementCount = 1800+ var statement strings.Builder+ payload := strings.Repeat("x", 5000)+ for i := 0; i < statementCount; i++ {+ fmt.Fprintf(&statement, "INSERT INTO t (id, v) VALUES (%d, '%s');\n", i, payload)+ }+ require.Greater(t, statement.Len(), common.MaxSheetCheckSize)++ commands, err := buildExecuteCommands(statement.String())+ require.NoError(t, err)+ require.Len(t, commands, statementCount)+ for _, command := range commands {+ require.Less(t, len(command.Text), common.MaxSheetCheckSize)+ }}diff --git a/backend/plugin/db/tidb/tidb.go b/backend/plugin/db/tidb/tidb.goindex a0250ed3bf..1fefeb21a3 100644--- a/backend/plugin/db/tidb/tidb.go+++ b/backend/plugin/db/tidb/tidb.go@@ -177,11 +177,11 @@ func parseVersion(version string) (string, error) {return "", errors.Errorf("failed to parse version %q", version)}+// buildExecuteCommands splits the statement into single commands.+// Commands are never merged back into one: a merged sheet is sent as a single+// COM_QUERY whose size equals the sheet size, tripping max_allowed_packet on+// large DML, and it collapses per-statement execution logging.func buildExecuteCommands(statement string) ([]base.Statement, error) {- if len(statement) > common.MaxSheetCheckSize {- return []base.Statement{{Text: statement}}, nil- }-commands, err := tidbparser.SplitSQL(statement)if err != nil {return nil, errors.Wrapf(err, "failed to split sql")@@ -190,9 +190,6 @@ func buildExecuteCommands(statement string) ([]base.Statement, error) {if len(commands) == 0 {return nil, nil}- if len(commands) > common.MaximumCommands {- return []base.Statement{{Text: statement}}, nil- }return commands, nil}diff --git a/backend/plugin/db/tidb/tidb_test.go b/backend/plugin/db/tidb/tidb_test.goindex 4b4f8f9036..e006d5f730 100644--- a/backend/plugin/db/tidb/tidb_test.go+++ b/backend/plugin/db/tidb/tidb_test.go@@ -1,6 +1,7 @@package tidbimport (+ "fmt""strings""testing"@@ -109,7 +110,7 @@ func TestBuildExecuteCommandsNormalizesDelimiter(t *testing.T) {require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")}-func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForTooManyCommands(t *testing.T) {+func TestBuildExecuteCommandsNormalizesDelimiterForManyCommands(t *testing.T) {var statement strings.Builderstatement.WriteString("DELIMITER //\n")statement.WriteString("CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND//\n")@@ -121,16 +122,39 @@ func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForTooManyCommands(t *testcommands, err := buildExecuteCommands(statement.String())require.NoError(t, err)- require.Len(t, commands, 1)- require.Equal(t, statement.String(), commands[0].Text)+ require.Greater(t, len(commands), common.MaximumCommands)+ require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")+ for _, command := range commands {+ require.NotContains(t, command.Text, "DELIMITER")+ }}-func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForLargeSheet(t *testing.T) {+func TestBuildExecuteCommandsNormalizesDelimiterForLargeSheet(t *testing.T) {statement := "DELIMITER //\nCREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND//\nDELIMITER ;\n" +strings.Repeat(" ", common.MaxSheetCheckSize)commands, err := buildExecuteCommands(statement)require.NoError(t, err)require.Len(t, commands, 1)- require.Equal(t, statement, commands[0].Text)+ require.NotContains(t, commands[0].Text, "DELIMITER")+ require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")+}++// A sheet over MaxSheetCheckSize used to be sent as one multi-statement packet,+// which trips the max_allowed_packet limit with error 1153 on large DML backfills.+func TestBuildExecuteCommandsSplitsOversizedSheet(t *testing.T) {+ const statementCount = 1800+ var statement strings.Builder+ payload := strings.Repeat("x", 5000)+ for i := 0; i < statementCount; i++ {+ fmt.Fprintf(&statement, "INSERT INTO t (id, v) VALUES (%d, '%s');\n", i, payload)+ }+ require.Greater(t, statement.Len(), common.MaxSheetCheckSize)++ commands, err := buildExecuteCommands(statement.String())+ require.NoError(t, err)+ require.Len(t, commands, statementCount)+ for _, command := range commands {+ require.Less(t, len(command.Text), common.MaxSheetCheckSize)+ }}