Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(mysql): produce valid SQL for CHECK constraints and routine headers in schema dumps (#20576)
rebelice9976c78605a1
diff --git a/backend/plugin/db/mysql/sync.go b/backend/plugin/db/mysql/sync.goindex 4d10fa16c7..870f7c3f6e 100644--- a/backend/plugin/db/mysql/sync.go+++ b/backend/plugin/db/mysql/sync.go@@ -416,6 +416,7 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetad); err != nil {return nil, err}+ check.Expression = unescapeCheckClause(check.Expression)key := db.TableKey{Schema: "", Table: tableName}checkMap[key] = append(checkMap[key], check)}@@ -906,17 +907,42 @@ func (d *Driver) getCreateFunctionStmt(ctx context.Context, databaseName, functif = fmt.Sprintf("CREATE%s", f[functionSymbolIdx:])}- if charsetIdx := strings.Index(f, " CHARSET "); charsetIdx != -1 {- if newLineIdx := strings.Index(f, "\n"); newLineIdx != -1 {- f = f[:charsetIdx] + f[newLineIdx:]- }- }+ f = stripReturnsCharset(f)return f, nil}return "", nil}+// unescapeCheckClause unescapes the check constraint clause stored in+// information_schema.CHECK_CONSTRAINTS.CHECK_CLAUSE, which escapes single+// quotes as \' (e.g. (`type` = _utf8mb4\'A\')). The escaped form is not valid+// SQL and must be unescaped before being written back into DDL.+func unescapeCheckClause(clause string) string {+ return strings.ReplaceAll(clause, `\'`, `'`)+}++// stripReturnsCharset removes the " CHARSET ... [COLLATE ...]" attributes that+// SHOW CREATE FUNCTION appends to the RETURNS clause. The search is anchored to+// the RETURNS line because the parameter list is preserved verbatim from the+// original definition: it can span multiple lines and may itself contain+// CHARSET attributes.+func stripReturnsCharset(def string) string {+ returnsIdx := strings.Index(def, " RETURNS ")+ if returnsIdx == -1 {+ return def+ }+ lineEnd := strings.Index(def[returnsIdx:], "\n")+ if lineEnd == -1 {+ return def+ }+ charsetIdx := strings.Index(def[returnsIdx:returnsIdx+lineEnd], " CHARSET ")+ if charsetIdx == -1 {+ return def+ }+ return def[:returnsIdx+charsetIdx] + def[returnsIdx+lineEnd:]+}+func (d *Driver) getCreateProcedureStmt(ctx context.Context, databaseName, functionName string) (string, error) {query := fmt.Sprintf("SHOW CREATE PROCEDURE `%s`.`%s`", databaseName, functionName)rows, err := d.db.QueryContext(ctx, query)diff --git a/backend/plugin/db/mysql/sync_test.go b/backend/plugin/db/mysql/sync_test.goindex ad20d75269..9158d08a4e 100644--- a/backend/plugin/db/mysql/sync_test.go+++ b/backend/plugin/db/mysql/sync_test.go@@ -444,3 +444,98 @@ func TestGetViewDefFromCreateView(t *testing.T) {require.Equal(t, tc.def, got)}}++func TestUnescapeCheckClause(t *testing.T) {+ testCases := []struct {+ clause string+ want string+ }{+ {+ // information_schema.CHECK_CONSTRAINTS escapes single quotes as \'.+ clause: `((` + "`type`" + ` = _utf8mb4\'A\') and (` + "`amount`" + ` is not null))`,+ want: "((`type` = _utf8mb4'A') and (`amount` is not null))",+ },+ {+ clause: "(`amount` > 0)",+ want: "(`amount` > 0)",+ },+ {+ clause: `(` + "`status`" + ` in (_utf8mb4\'open\',_utf8mb4\'closed\'))`,+ want: "(`status` in (_utf8mb4'open',_utf8mb4'closed'))",+ },+ }++ for _, tc := range testCases {+ require.Equal(t, tc.want, unescapeCheckClause(tc.clause))+ }+}++func TestStripReturnsCharset(t *testing.T) {+ testCases := []struct {+ name string+ def string+ want string+ }{+ {+ name: "multiline parameter list with RETURNS CHARSET and COLLATE",+ def: "CREATE FUNCTION `f1`(\n" ++ " p_a BIGINT,\n" ++ " p_b VARCHAR(36)\n" ++ ") RETURNS char(36) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci\n" ++ " NO SQL\n" ++ " DETERMINISTIC\n" ++ "BEGIN\n" ++ " RETURN p_b;\n" ++ "END",+ want: "CREATE FUNCTION `f1`(\n" ++ " p_a BIGINT,\n" ++ " p_b VARCHAR(36)\n" ++ ") RETURNS char(36)\n" ++ " NO SQL\n" ++ " DETERMINISTIC\n" ++ "BEGIN\n" ++ " RETURN p_b;\n" ++ "END",+ },+ {+ name: "single-line parameter list with RETURNS CHARSET",+ def: "CREATE FUNCTION `f2`(i INT) RETURNS char(36) CHARSET utf8mb4\n" ++ "BEGIN\n" ++ " RETURN 'x';\n" ++ "END",+ want: "CREATE FUNCTION `f2`(i INT) RETURNS char(36)\n" ++ "BEGIN\n" ++ " RETURN 'x';\n" ++ "END",+ },+ {+ name: "RETURNS without CHARSET is unchanged",+ def: "CREATE FUNCTION `f3`(i INT) RETURNS int\n" ++ "RETURN i + 1",+ want: "CREATE FUNCTION `f3`(i INT) RETURNS int\n" ++ "RETURN i + 1",+ },+ {+ name: "CHARSET in parameter list only is unchanged",+ def: "CREATE FUNCTION `f4`(\n" ++ " p_a VARCHAR(10) CHARSET utf8mb4,\n" ++ " p_b INT\n" ++ ") RETURNS int\n" ++ "RETURN p_b",+ want: "CREATE FUNCTION `f4`(\n" ++ " p_a VARCHAR(10) CHARSET utf8mb4,\n" ++ " p_b INT\n" ++ ") RETURNS int\n" ++ "RETURN p_b",+ },+ {+ name: "no RETURNS clause is unchanged",+ def: "CREATE PROCEDURE `p1`(IN a INT)\nBEGIN\nEND",+ want: "CREATE PROCEDURE `p1`(IN a INT)\nBEGIN\nEND",+ },+ }++ for _, tc := range testCases {+ require.Equal(t, tc.want, stripReturnsCharset(tc.def), tc.name)+ }+}diff --git a/backend/plugin/schema/mysql/get_database_definition_test.go b/backend/plugin/schema/mysql/get_database_definition_test.gonew file mode 100644index 0000000000..eb0f4eceea--- /dev/null+++ b/backend/plugin/schema/mysql/get_database_definition_test.go@@ -0,0 +1,87 @@+package mysql++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++func TestGetTableDefinitionWithCheckConstraint(t *testing.T) {+ table := &storepb.TableMetadata{+ Name: "t1",+ Columns: []*storepb.ColumnMetadata{+ {Name: "type", Type: "varchar(10)", Nullable: true, Default: "NULL"},+ {Name: "amount", Type: "decimal(10,2)", Nullable: true, Default: "NULL"},+ },+ CheckConstraints: []*storepb.CheckConstraintMetadata{+ {+ Name: "c1",+ // The expression as synced from MySQL after unescaping the+ // information_schema escaped form ((`type` = _utf8mb4\'A\')...).+ Expression: "(((`type` = _utf8mb4'A') and (`amount` is not null)))",+ },+ },+ Engine: "InnoDB",+ Charset: "utf8mb4",+ Collation: "utf8mb4_general_ci",+ }++ got, err := GetTableDefinition("", table, nil)+ require.NoError(t, err)++ want := "--\n" ++ "-- Table structure for `t1`\n" ++ "--\n" ++ "CREATE TABLE `t1` (\n" ++ " `type` varchar(10) DEFAULT NULL,\n" ++ " `amount` decimal(10,2) DEFAULT NULL,\n" ++ " CONSTRAINT `c1` CHECK (((`type` = _utf8mb4'A') and (`amount` is not null)))\n" ++ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;\n\n"+ require.Equal(t, want, got)+ require.NotContains(t, got, `\'`)+}++func TestGetFunctionDefinitionWithMultilineParameters(t *testing.T) {+ function := &storepb.FunctionMetadata{+ Name: "f1",+ // The definition as synced from SHOW CREATE FUNCTION: the parameter+ // list keeps its original multi-line formatting, and the CHARSET and+ // COLLATE attributes of the RETURNS clause have been stripped.+ Definition: "CREATE FUNCTION `f1`(\n" ++ " p_a BIGINT,\n" ++ " p_b VARCHAR(36)\n" ++ ") RETURNS char(36)\n" ++ " NO SQL\n" ++ " DETERMINISTIC\n" ++ "BEGIN\n" ++ " RETURN p_b;\n" ++ "END",+ CharacterSetClient: "utf8mb4",+ CollationConnection: "utf8mb4_unicode_ci",+ SqlMode: "ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES",+ }++ got, err := GetFunctionDefinition("", function)+ require.NoError(t, err)++ want := "--\n" ++ "-- Function structure for `f1`\n" ++ "--\n" ++ "SET character_set_client = utf8mb4;\n" ++ "SET character_set_results = utf8mb4;\n" ++ "SET collation_connection = utf8mb4_unicode_ci;\n" ++ "SET sql_mode = 'ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES';\n" ++ "CREATE FUNCTION `f1`(\n" ++ " p_a BIGINT,\n" ++ " p_b VARCHAR(36)\n" ++ ") RETURNS char(36)\n" ++ " NO SQL\n" ++ " DETERMINISTIC\n" ++ "BEGIN\n" ++ " RETURN p_b;\n" ++ "END;;\n" ++ "DELIMITER ;\n\n"+ require.Equal(t, want, got)+}diff --git a/backend/plugin/schema/mysql/get_database_definition_testcontainer_test.go b/backend/plugin/schema/mysql/get_database_definition_testcontainer_test.goindex 775f485cf8..a068599341 100644--- a/backend/plugin/schema/mysql/get_database_definition_testcontainer_test.go+++ b/backend/plugin/schema/mysql/get_database_definition_testcontainer_test.go@@ -283,6 +283,31 @@ CREATE TABLE translations (translation TEXT CHARACTER SET utf8mb4 COLLATE utf8mb4_unicode_ci,UNIQUE KEY uk_lang_key (language_code, content_key)) DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;+`,+ },+ {+ description: "Check constraints with string literals and function with multiline parameters",+ originalDDL: `+CREATE TABLE transactions (+ id INT PRIMARY KEY AUTO_INCREMENT,+ type VARCHAR(10),+ amount DECIMAL(10, 2),+ CONSTRAINT c1 CHECK (type = 'A' AND amount IS NOT NULL)+);++DELIMITER $$++CREATE FUNCTION lookup_token(+ p_a BIGINT,+ p_b VARCHAR(36)+) RETURNS CHAR(36) CHARSET utf8mb4 COLLATE utf8mb4_unicode_ci+ NO SQL+ DETERMINISTIC+BEGIN+ RETURN p_b;+END$$++DELIMITER ;`,},}diff --git a/backend/plugin/schema/mysql/get_database_metadata_testcontainer_test.go b/backend/plugin/schema/mysql/get_database_metadata_testcontainer_test.goindex b84cee9683..49d375b27a 100644--- a/backend/plugin/schema/mysql/get_database_metadata_testcontainer_test.go+++ b/backend/plugin/schema/mysql/get_database_metadata_testcontainer_test.go@@ -846,9 +846,10 @@ func normalizeCheckExpression(expr string) string {expr = strings.ReplaceAll(expr, "`", "")expr = strings.ReplaceAll(expr, " ", "")- // Remove character set prefixes like _utf8mb4- re := regexp.MustCompile(`_[a-zA-Z0-9]+\\`)- expr = re.ReplaceAllString(expr, "")+ // Remove character set introducers attached to string literals, like+ // _utf8mb4'...' (synced metadata) or _utf8mb4\'...\' (legacy escaped form)+ re := regexp.MustCompile(`_[a-zA-Z0-9]+\\?'`)+ expr = re.ReplaceAllString(expr, "'")// Remove escaped quotesexpr = strings.ReplaceAll(expr, `\'`, `'`)