Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(mysql,tidb): always split statements instead of merging large sheets

Vincent Huang395e5d075d7c
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.go
index f34cc07bed..fc524b8e49 100644
--- a/backend/plugin/db/mysql/mysql.go
+++ b/backend/plugin/db/mysql/mysql.go
@@ -307,11 +307,11 @@ func parseVersion(version string) (string, string, error) {
return "", "", errors.Errorf("failed to parse version %q", version)
}
+// buildExecuteCommands splits the statement into single commands.
+// Commands are never merged back into one: a merged sheet is sent as a single
+// COM_QUERY whose size equals the sheet size, tripping max_allowed_packet on
+// large DML, and it collapses per-statement execution logging.
func buildExecuteCommands(statement string) ([]base.Statement, error) {
- if len(statement) > common.MaxSheetCheckSize {
- return []base.Statement{{Text: statement}}, nil
- }
-
commands, err := mysqlparser.SplitSQL(statement)
if err != nil {
return nil, errors.Wrapf(err, "failed to split sql")
@@ -320,9 +320,6 @@ func buildExecuteCommands(statement string) ([]base.Statement, error) {
if len(commands) == 0 {
return nil, nil
}
- if len(commands) > common.MaximumCommands {
- return []base.Statement{{Text: statement}}, nil
- }
return commands, nil
}
diff --git a/backend/plugin/db/mysql/mysql_test.go b/backend/plugin/db/mysql/mysql_test.go
index 32d326bdf5..4b38d14d9a 100644
--- a/backend/plugin/db/mysql/mysql_test.go
+++ b/backend/plugin/db/mysql/mysql_test.go
@@ -1,6 +1,7 @@
package mysql
import (
+ "fmt"
"strings"
"testing"
@@ -127,7 +128,7 @@ func TestBuildExecuteCommandsNormalizesDelimiter(t *testing.T) {
require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")
}
-func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForTooManyCommands(t *testing.T) {
+func TestBuildExecuteCommandsNormalizesDelimiterForManyCommands(t *testing.T) {
var statement strings.Builder
statement.WriteString("DELIMITER //\n")
statement.WriteString("CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND//\n")
@@ -139,16 +140,39 @@ func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForTooManyCommands(t *test
commands, err := buildExecuteCommands(statement.String())
require.NoError(t, err)
- require.Len(t, commands, 1)
- require.Equal(t, statement.String(), commands[0].Text)
+ require.Greater(t, len(commands), common.MaximumCommands)
+ require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")
+ for _, command := range commands {
+ require.NotContains(t, command.Text, "DELIMITER")
+ }
}
-func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForLargeSheet(t *testing.T) {
+func TestBuildExecuteCommandsNormalizesDelimiterForLargeSheet(t *testing.T) {
statement := "DELIMITER //\nCREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND//\nDELIMITER ;\n" +
strings.Repeat(" ", common.MaxSheetCheckSize)
commands, err := buildExecuteCommands(statement)
require.NoError(t, err)
require.Len(t, commands, 1)
- require.Equal(t, statement, commands[0].Text)
+ require.NotContains(t, commands[0].Text, "DELIMITER")
+ require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")
+}
+
+// A sheet over MaxSheetCheckSize used to be sent as one multi-statement packet,
+// which trips MySQL's max_allowed_packet with error 1153 on large DML backfills.
+func TestBuildExecuteCommandsSplitsOversizedSheet(t *testing.T) {
+ const statementCount = 1800
+ var statement strings.Builder
+ payload := strings.Repeat("x", 5000)
+ for i := 0; i < statementCount; i++ {
+ fmt.Fprintf(&statement, "INSERT INTO t (id, v) VALUES (%d, '%s');\n", i, payload)
+ }
+ require.Greater(t, statement.Len(), common.MaxSheetCheckSize)
+
+ commands, err := buildExecuteCommands(statement.String())
+ require.NoError(t, err)
+ require.Len(t, commands, statementCount)
+ for _, command := range commands {
+ require.Less(t, len(command.Text), common.MaxSheetCheckSize)
+ }
}
diff --git a/backend/plugin/db/tidb/tidb.go b/backend/plugin/db/tidb/tidb.go
index a0250ed3bf..1fefeb21a3 100644
--- a/backend/plugin/db/tidb/tidb.go
+++ b/backend/plugin/db/tidb/tidb.go
@@ -177,11 +177,11 @@ func parseVersion(version string) (string, error) {
return "", errors.Errorf("failed to parse version %q", version)
}
+// buildExecuteCommands splits the statement into single commands.
+// Commands are never merged back into one: a merged sheet is sent as a single
+// COM_QUERY whose size equals the sheet size, tripping max_allowed_packet on
+// large DML, and it collapses per-statement execution logging.
func buildExecuteCommands(statement string) ([]base.Statement, error) {
- if len(statement) > common.MaxSheetCheckSize {
- return []base.Statement{{Text: statement}}, nil
- }
-
commands, err := tidbparser.SplitSQL(statement)
if err != nil {
return nil, errors.Wrapf(err, "failed to split sql")
@@ -190,9 +190,6 @@ func buildExecuteCommands(statement string) ([]base.Statement, error) {
if len(commands) == 0 {
return nil, nil
}
- if len(commands) > common.MaximumCommands {
- return []base.Statement{{Text: statement}}, nil
- }
return commands, nil
}
diff --git a/backend/plugin/db/tidb/tidb_test.go b/backend/plugin/db/tidb/tidb_test.go
index 4b4f8f9036..e006d5f730 100644
--- a/backend/plugin/db/tidb/tidb_test.go
+++ b/backend/plugin/db/tidb/tidb_test.go
@@ -1,6 +1,7 @@
package tidb
import (
+ "fmt"
"strings"
"testing"
@@ -109,7 +110,7 @@ func TestBuildExecuteCommandsNormalizesDelimiter(t *testing.T) {
require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")
}
-func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForTooManyCommands(t *testing.T) {
+func TestBuildExecuteCommandsNormalizesDelimiterForManyCommands(t *testing.T) {
var statement strings.Builder
statement.WriteString("DELIMITER //\n")
statement.WriteString("CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND//\n")
@@ -121,16 +122,39 @@ func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForTooManyCommands(t *test
commands, err := buildExecuteCommands(statement.String())
require.NoError(t, err)
- require.Len(t, commands, 1)
- require.Equal(t, statement.String(), commands[0].Text)
+ require.Greater(t, len(commands), common.MaximumCommands)
+ require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")
+ for _, command := range commands {
+ require.NotContains(t, command.Text, "DELIMITER")
+ }
}
-func TestBuildExecuteCommandsDoesNotNormalizeDelimiterForLargeSheet(t *testing.T) {
+func TestBuildExecuteCommandsNormalizesDelimiterForLargeSheet(t *testing.T) {
statement := "DELIMITER //\nCREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND//\nDELIMITER ;\n" +
strings.Repeat(" ", common.MaxSheetCheckSize)
commands, err := buildExecuteCommands(statement)
require.NoError(t, err)
require.Len(t, commands, 1)
- require.Equal(t, statement, commands[0].Text)
+ require.NotContains(t, commands[0].Text, "DELIMITER")
+ require.Contains(t, commands[0].Text, "CREATE PROCEDURE p()\nBEGIN\n SELECT 1;\nEND")
+}
+
+// A sheet over MaxSheetCheckSize used to be sent as one multi-statement packet,
+// which trips the max_allowed_packet limit with error 1153 on large DML backfills.
+func TestBuildExecuteCommandsSplitsOversizedSheet(t *testing.T) {
+ const statementCount = 1800
+ var statement strings.Builder
+ payload := strings.Repeat("x", 5000)
+ for i := 0; i < statementCount; i++ {
+ fmt.Fprintf(&statement, "INSERT INTO t (id, v) VALUES (%d, '%s');\n", i, payload)
+ }
+ require.Greater(t, statement.Len(), common.MaxSheetCheckSize)
+
+ commands, err := buildExecuteCommands(statement.String())
+ require.NoError(t, err)
+ require.Len(t, commands, statementCount)
+ for _, command := range commands {
+ require.Less(t, len(command.Text), common.MaxSheetCheckSize)
+ }
}