Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(mariadb): sync sequences and system-versioned tables (#21665)
rebelicea72a8b87c48e
diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.goindex 8be82c1ab6..d9910e420d 100644--- a/backend/common/testcontainer/shared.go+++ b/backend/common/testcontainer/shared.go@@ -23,6 +23,9 @@ var (sharedMSSQL = startOnce(getMSSQLContainer)sharedTiDB = startOnce(getTiDBContainer)sharedMongo = startOnce(getMongoDBContainer)+ sharedMariaDB = startOnce(mariaDBStarter("mariadb:11.8.8"))+ // 10.7 is the oldest MariaDB release Bytebase supports.+ sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))startedMu sync.Mutexstarted []*Container@@ -103,6 +106,14 @@ func SharedTiDBContainer(t testing.TB) *Container { return shared(t, sharedTiDB)// each with NewMongoDatabase.func SharedMongoDBContainer(t testing.TB) *Container { return shared(t, sharedMongo) }+// SharedMariaDBContainer is SharedPgContainer for MariaDB 11.8; tests take a+// database each with NewMariaDBDatabase.+func SharedMariaDBContainer(t testing.TB) *Container { return shared(t, sharedMariaDB) }++// SharedMariaDB107Container is SharedMariaDBContainer for MariaDB 10.7, the+// oldest supported release; tests take a database each with NewMariaDB107Database.+func SharedMariaDB107Container(t testing.TB) *Container { return shared(t, sharedMariaDB107) }+// NewPgDatabase creates an empty database for the test on the shared// PostgreSQL container and returns its name with a superuser handle to it.// Anything cluster-wide the test creates, such as a role, should be named@@ -139,7 +150,24 @@ func createPgDatabase(t testing.TB, container *Container) (string, *sql.DB) {// container and returns the container with the database's name.func NewTiDBDatabase(t testing.TB) (*Container, string) {t.Helper()- container := SharedTiDBContainer(t)+ return createMySQLProtocolDatabase(t, SharedTiDBContainer(t))+}++// NewMariaDBDatabase creates an empty database for the test on the shared+// MariaDB container and returns the container with the database's name.+func NewMariaDBDatabase(t testing.TB) (*Container, string) {+ t.Helper()+ return createMySQLProtocolDatabase(t, SharedMariaDBContainer(t))+}++// NewMariaDB107Database is NewMariaDBDatabase on the shared MariaDB 10.7 container.+func NewMariaDB107Database(t testing.TB) (*Container, string) {+ t.Helper()+ return createMySQLProtocolDatabase(t, SharedMariaDB107Container(t))+}++func createMySQLProtocolDatabase(t testing.TB, container *Container) (*Container, string) {+ t.Helper()name := newDatabaseName()_, err := container.GetDB().Exec("CREATE DATABASE " + name)require.NoError(t, err)diff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.goindex 60432aecf4..8bace1dbf8 100644--- a/backend/common/testcontainer/testcontainer.go+++ b/backend/common/testcontainer/testcontainer.go@@ -243,6 +243,26 @@ func GetMySQLContainer(ctx context.Context) (*Container, error) {})}+// mariaDBStarter starts the MariaDB image, whose root password is root-password.+func mariaDBStarter(image string) func(context.Context) (*Container, error) {+ return func(ctx context.Context) (*Container, error) {+ const user, password = "root", "root-password"+ return start(ctx, testcontainers.ContainerRequest{+ Image: image,+ Env: map[string]string{"MARIADB_ROOT_PASSWORD": password},+ WaitingFor: wait.ForLog("ready for connections").WithOccurrence(2).WithStartupTimeout(5 * time.Minute),+ }, spec{+ port: "3306/tcp",+ username: user,+ password: password,+ driver: "mysql",+ dsn: func(c *Container) string {+ return fmt.Sprintf("%s:%s@tcp(%s:%s)/?multiStatements=true", c.username, c.password, c.host, c.port)+ },+ })+ }+}+// mssqlTLSConf points SQL Server at the certificate getMSSQLContainer copies in,// leaving encryption up to the client.const mssqlTLSConf = `[network]diff --git a/backend/plugin/db/mysql/main_test.go b/backend/plugin/db/mysql/main_test.gonew file mode 100644index 0000000000..0a9c4993be--- /dev/null+++ b/backend/plugin/db/mysql/main_test.go@@ -0,0 +1,9 @@+package mysql++import (+ "testing"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+)++func TestMain(m *testing.M) { testcontainer.Main(m) }diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.goindex 58a2d876de..7bcd0f82d9 100644--- a/backend/plugin/db/mysql/mysql.go+++ b/backend/plugin/db/mysql/mysql.go@@ -40,6 +40,9 @@ import (var (baseTableType = "BASE TABLE"viewTableType = "VIEW"+ // MariaDB-only TABLE_TYPE values. Both kinds share the table namespace.+ mariaDBSystemVersionedTableType = "SYSTEM VERSIONED"+ mariaDBSequenceTableType = "SEQUENCE"_ db.Driver = (*Driver)(nil))diff --git a/backend/plugin/db/mysql/sync.go b/backend/plugin/db/mysql/sync.goindex 6fbef7422c..9985ed4724 100644--- a/backend/plugin/db/mysql/sync.go+++ b/backend/plugin/db/mysql/sync.go@@ -583,6 +583,10 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMereturn nil, util.FormatErrorWithQuery(err, tableQuery)}defer tableRows.Close()+ // Keyed on the registered engine, not the server: an instance registered as MySQL+ // keeps its MySQL sync result even when the server is MariaDB.+ mariaDBInstance := d.dbType == storepb.Engine_MARIADB+ var sequences []*metadatapb.SequenceMetadatafor tableRows.Next() {var tableName, tableType, engine, collation, createOptions, comment, charset stringvar rowCount, dataSize, indexSize, dataFree int64@@ -612,8 +616,11 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe// The partition details are fetched separately by listPartitionTables.key := db.TableKey{Schema: "", Table: tableName}- switch tableType {- case baseTableType:+ // A MariaDB system-versioned table is synced as a table: its hidden row_start+ // and row_end columns are absent from information_schema.COLUMNS, and explicit+ // ones arrive as stored generated columns whose expression is ROW START/ROW END.+ switch {+ case tableType == baseTableType, mariaDBInstance && tableType == mariaDBSystemVersionedTableType:columns := columnMap[key]tableMetadata := &metadatapb.TableMetadata{Name: tableName,@@ -635,6 +642,11 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMeif tableCollation.Valid {tableMetadata.Collation = tableCollation.String}+ if tableType == mariaDBSystemVersionedTableType {+ // Recorded as the table option SHOW CREATE TABLE prints; the DDL writers in+ // backend/plugin/schema/mysql look for it to keep the versioning.+ tableMetadata.CreateOptions = strings.TrimSpace(createOptions + " WITH SYSTEM VERSIONING")+ }var indexNames []stringif indexes, ok := indexMap[key]; ok {for indexName := range indexes {@@ -647,10 +659,16 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe}schemaMetadata.Tables = append(schemaMetadata.Tables, tableMetadata)- case viewTableType:+ case tableType == viewTableType:if view, ok := viewMap[key]; ok {schemaMetadata.Views = append(schemaMetadata.Views, view)}+ case mariaDBInstance && tableType == mariaDBSequenceTableType:+ sequences = append(sequences, &metadatapb.SequenceMetadata{+ Name: tableName,+ DataType: mariaDBSequenceDataType(columnMap[key]),+ Comment: comment,+ })default:// Unknown table type, skip}@@ -658,6 +676,10 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMeif err := tableRows.Err(); err != nil {return nil, util.FormatErrorWithQuery(err, tableQuery)}+ if err := d.readMariaDBSequenceAttributes(ctx, d.databaseName, sequences); err != nil {+ return nil, err+ }+ schemaMetadata.Sequences = sequencesdatabaseMetadata := &metadatapb.DatabaseSchemaMetadata{Name: d.databaseName,@@ -683,6 +705,49 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMereturn databaseMetadata, err}+var displayWidthRegexp = regexp.MustCompile(`\(\d+\)`)++// mariaDBSequenceDataType derives a sequence's AS type from its start_value column:+// with the display width removed, the column type spells the type the way+// information_schema.SEQUENCES.DATA_TYPE does (MariaDB 11.5+), e.g. "int unsigned".+func mariaDBSequenceDataType(columns []*metadatapb.ColumnMetadata) string {+ for _, column := range columns {+ if column.Name == "start_value" {+ return displayWidthRegexp.ReplaceAllString(column.Type, "")+ }+ }+ return ""+}++// readMariaDBSequenceAttributes reads each sequence's own row. information_schema.SEQUENCES+// is no substitute: it exists only from MariaDB 11.5 and has no cache size. A sequence whose+// row cannot be read, such as one the user has no SELECT privilege on, keeps only its name.+func (d *Driver) readMariaDBSequenceAttributes(ctx context.Context, databaseName string, sequences []*metadatapb.SequenceMetadata) error {+ for _, sequence := range sequences {+ query := fmt.Sprintf(+ "SELECT start_value, minimum_value, maximum_value, increment, cache_size, cycle_option FROM `%s`.`%s`",+ strings.ReplaceAll(databaseName, "`", "``"),+ strings.ReplaceAll(sequence.Name, "`", "``"),+ )+ var start, minValue, maxValue, increment, cacheSize string+ var cycle bool+ if err := d.db.QueryRowContext(ctx, query).Scan(&start, &minValue, &maxValue, &increment, &cacheSize, &cycle); err != nil {+ if ctx.Err() != nil {+ return ctx.Err()+ }+ slog.Warn("failed to read MariaDB sequence attributes", slog.String("database", databaseName), slog.String("sequence", sequence.Name), log.BBError(err))+ continue+ }+ sequence.Start = start+ sequence.MinValue = minValue+ sequence.MaxValue = maxValue+ sequence.Increment = increment+ sequence.CacheSize = cacheSize+ sequence.Cycle = cycle+ }+ return nil+}+func (d *Driver) getEventList(ctx context.Context, databaseName string) ([]*metadatapb.EventMetadata, error) {listEventsQuery := `SELECTdiff --git a/backend/plugin/db/mysql/sync_test.go b/backend/plugin/db/mysql/sync_test.goindex 969d71f0d3..e5c89421bf 100644--- a/backend/plugin/db/mysql/sync_test.go+++ b/backend/plugin/db/mysql/sync_test.go@@ -1,12 +1,21 @@package mysqlimport (+ "context""database/sql"+ "fmt"+ "maps"+ "slices""strings""testing"metadatapb "github.com/bytebase/omni/metadata"+ "github.com/google/go-cmp/cmp""github.com/stretchr/testify/require"+ "google.golang.org/protobuf/testing/protocmp"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store")func TestSetColumnMetadataDefault(t *testing.T) {@@ -902,3 +911,130 @@ func TestStripReturnsCharset(t *testing.T) {require.Equal(t, tc.want, stripReturnsCharset(tc.def), tc.name)}}++// TestSyncMariaDBTableNamespace covers the MariaDB objects that share the table+// namespace with base tables. A consumer can only conclude that a table name is+// free if sync lists every one of them.+func TestSyncMariaDBTableNamespace(t *testing.T) {+ t.Parallel()+ for _, tc := range []struct {+ version string+ newDatabase func(testing.TB) (*testcontainer.Container, string)+ }{+ {version: "11.8", newDatabase: testcontainer.NewMariaDBDatabase},+ {version: "10.7", newDatabase: testcontainer.NewMariaDB107Database},+ } {+ t.Run(tc.version, func(t *testing.T) {+ t.Parallel()+ ctx := context.Background()+ container, database := tc.newDatabase(t)+ driver := openTestDriver(ctx, t, storepb.Engine_MARIADB, container, container.GetUsername(), container.GetPassword(), database)+ _, err := driver.db.ExecContext(ctx, `+CREATE TABLE parent (id INT PRIMARY KEY);+CREATE TABLE history (+ id INT PRIMARY KEY,+ amount INT,+ parent_id INT,+ KEY idx_amount (amount),+ CONSTRAINT fk_parent FOREIGN KEY (parent_id) REFERENCES parent (id),+ CONSTRAINT chk_amount CHECK (amount > 0)+) WITH SYSTEM VERSIONING COMMENT 'versioned';+CREATE TRIGGER trg_history BEFORE INSERT ON history FOR EACH ROW SET NEW.amount = NEW.amount;+CREATE TABLE explicit_period (+ id INT,+ valid_from TIMESTAMP(6) GENERATED ALWAYS AS ROW START,+ valid_to TIMESTAMP(6) GENERATED ALWAYS AS ROW END,+ PERIOD FOR SYSTEM_TIME (valid_from, valid_to)+) WITH SYSTEM VERSIONING;+CREATE SEQUENCE seq_default;+CREATE SEQUENCE seq_custom START WITH 100 INCREMENT BY 5 MINVALUE 10 MAXVALUE 1000 CACHE 20 CYCLE COMMENT 'sequence comment';+`)+ require.NoError(t, err)++ metadata, err := driver.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Len(t, metadata.Schemas, 1)+ schema := metadata.Schemas[0]++ tables := make(map[string]*metadatapb.TableMetadata)+ for _, table := range schema.Tables {+ tables[table.Name] = table+ }+ require.ElementsMatch(t, []string{"explicit_period", "history", "parent"}, slices.Collect(maps.Keys(tables)))++ history := tables["history"]+ require.Equal(t, []string{"id", "amount", "parent_id"}, columnNames(history.Columns))+ var indexNames []string+ for _, index := range history.Indexes {+ indexNames = append(indexNames, index.Name)+ }+ require.Equal(t, []string{"PRIMARY", "fk_parent", "idx_amount"}, indexNames)+ require.Equal(t, []string{"id"}, history.Indexes[0].Expressions)+ require.Len(t, history.ForeignKeys, 1)+ require.Equal(t, "fk_parent", history.ForeignKeys[0].Name)+ require.Equal(t, "parent", history.ForeignKeys[0].ReferencedTable)+ require.Empty(t, cmp.Diff([]*metadatapb.CheckConstraintMetadata{{Name: "chk_amount", Expression: "`amount` > 0"}}, history.CheckConstraints, protocmp.Transform()))+ require.Len(t, history.Triggers, 1)+ require.Equal(t, "trg_history", history.Triggers[0].Name)+ require.Equal(t, "versioned", history.Comment)+ require.Equal(t, "WITH SYSTEM VERSIONING", history.CreateOptions)+ require.Empty(t, tables["parent"].CreateOptions)++ explicitPeriod := tables["explicit_period"]+ require.Equal(t, []string{"id", "valid_from", "valid_to"}, columnNames(explicitPeriod.Columns))+ require.Equal(t, "WITH SYSTEM VERSIONING", explicitPeriod.CreateOptions)+ require.Empty(t, cmp.Diff(&metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW START"}, explicitPeriod.Columns[1].Generation, protocmp.Transform()))+ require.Empty(t, cmp.Diff(&metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW END"}, explicitPeriod.Columns[2].Generation, protocmp.Transform()))++ require.Empty(t, cmp.Diff([]*metadatapb.SequenceMetadata{+ {Name: "seq_custom", DataType: "bigint", Start: "100", MinValue: "10", MaxValue: "1000", Increment: "5", CacheSize: "20", Cycle: true, Comment: "sequence comment"},+ {Name: "seq_default", DataType: "bigint", Start: "1", MinValue: "1", MaxValue: "9223372036854775806", Increment: "1", CacheSize: "1000"},+ }, schema.Sequences, protocmp.Transform()))++ // The same server registered as a MySQL instance keeps the MySQL sync result.+ asMySQL := openTestDriver(ctx, t, storepb.Engine_MYSQL, container, container.GetUsername(), container.GetPassword(), database)+ metadata, err = asMySQL.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Len(t, metadata.Schemas, 1)+ require.Len(t, metadata.Schemas[0].Tables, 1)+ require.Equal(t, "parent", metadata.Schemas[0].Tables[0].Name)+ require.Empty(t, metadata.Schemas[0].Sequences)+ })+ }+}++func TestSyncMariaDBSequenceAttributes(t *testing.T) {+ t.Parallel()+ ctx := context.Background()+ container, database := testcontainer.NewMariaDBDatabase(t)+ reader := database + "_reader"+ _, err := container.GetDB().ExecContext(ctx, fmt.Sprintf(`+CREATE SEQUENCE %[1]s.seq_typed AS INT UNSIGNED START WITH 7 MAXVALUE 4000000000;+CREATE SEQUENCE %[1]s.seq_unreadable;+CREATE USER '%[2]s'@'%%' IDENTIFIED BY 'reader-password';+GRANT SELECT ON %[1]s.seq_typed TO '%[2]s'@'%%';+GRANT INSERT ON %[1]s.seq_unreadable TO '%[2]s'@'%%';+`, database, reader))+ require.NoError(t, err)++ driver := openTestDriver(ctx, t, storepb.Engine_MARIADB, container, reader, "reader-password", database)+ metadata, err := driver.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Len(t, metadata.Schemas, 1)+ schema := metadata.Schemas[0]++ require.Empty(t, schema.Tables)+ // Without SELECT on a sequence its row is unreadable, but the name is still taken.+ require.Empty(t, cmp.Diff([]*metadatapb.SequenceMetadata{+ {Name: "seq_typed", DataType: "int unsigned", Start: "7", MinValue: "1", MaxValue: "4000000000", Increment: "1", CacheSize: "1000"},+ {Name: "seq_unreadable", DataType: "bigint"},+ }, schema.Sequences, protocmp.Transform()))+}++func columnNames(columns []*metadatapb.ColumnMetadata) []string {+ var names []string+ for _, column := range columns {+ names = append(names, column.Name)+ }+ return names+}diff --git a/backend/plugin/db/mysql/testutil_test.go b/backend/plugin/db/mysql/testutil_test.gonew file mode 100644index 0000000000..37d985ca98--- /dev/null+++ b/backend/plugin/db/mysql/testutil_test.go@@ -0,0 +1,33 @@+package mysql++import (+ "context"+ "testing"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+)++func openTestDriver(ctx context.Context, t *testing.T, engine storepb.Engine, container *testcontainer.Container, username, password, database string) *Driver {+ t.Helper()++ d, err := newDriver().Open(ctx, engine, db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Type: storepb.DataSourceType_ADMIN,+ Username: username,+ Host: container.GetHost(),+ Port: container.GetPort(),+ },+ Password: password,+ ConnectionContext: db.ConnectionContext{DatabaseName: database},+ })+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, d.Close(ctx)) })++ driver, ok := d.(*Driver)+ require.True(t, ok)+ return driver+}diff --git a/backend/plugin/schema/mysql/generate_migration.go b/backend/plugin/schema/mysql/generate_migration.goindex 1a2acfb895..063955daaf 100644--- a/backend/plugin/schema/mysql/generate_migration.go+++ b/backend/plugin/schema/mysql/generate_migration.go@@ -546,6 +546,7 @@ func writeCreateTableWithoutForeignKeys(buf *strings.Builder, tableName string,// generated-column ordering: GENERATED ALWAYS AS (...) precedes NOT NULL/SRID).writeColumnDefinitionBody(buf, col)}+ writeSystemTimePeriod(buf, table)// Write primary key constraint inline if existsfor _, index := range table.Indexes {@@ -611,6 +612,7 @@ func writeCreateTableWithoutForeignKeys(buf *strings.Builder, tableName string,_, _ = buf.WriteString(table.Comment)_, _ = buf.WriteString("'")}+ writeSystemVersioningOption(buf, table)// Partitioning is part of the table definition, so it has to be emitted here: MySQL// cannot partition a table after the fact without rewriting it. printPartitionClause@@ -708,17 +710,21 @@ func writeColumnDefinitionBody(buf *strings.Builder, column *metadatapb.ColumnMe// and generated columns. Generated columns never carry DEFAULT / AUTO_INCREMENT / ON// UPDATE, so those clauses are intentionally omitted.func writeGeneratedColumnAttributes(buf *strings.Builder, column *metadatapb.ColumnMetadata) {- _, _ = buf.WriteString(" GENERATED ALWAYS AS (")- _, _ = buf.WriteString(column.Generation.Expression)- _, _ = buf.WriteString(") ")- switch column.Generation.Type {- case metadatapb.GenerationMetadata_TYPE_STORED:- _, _ = buf.WriteString("STORED")- case metadatapb.GenerationMetadata_TYPE_VIRTUAL:- _, _ = buf.WriteString("VIRTUAL")- default:- // Default to VIRTUAL for unknown types- _, _ = buf.WriteString("VIRTUAL")+ if isSystemTimeColumn(column) {+ writeSystemTimeColumnAttribute(buf, column)+ } else {+ _, _ = buf.WriteString(" GENERATED ALWAYS AS (")+ _, _ = buf.WriteString(column.Generation.Expression)+ _, _ = buf.WriteString(") ")+ switch column.Generation.Type {+ case metadatapb.GenerationMetadata_TYPE_STORED:+ _, _ = buf.WriteString("STORED")+ case metadatapb.GenerationMetadata_TYPE_VIRTUAL:+ _, _ = buf.WriteString("VIRTUAL")+ default:+ // Default to VIRTUAL for unknown types+ _, _ = buf.WriteString("VIRTUAL")+ }}if !column.Nullable {diff --git a/backend/plugin/schema/mysql/generate_migration_test.go b/backend/plugin/schema/mysql/generate_migration_test.goindex 88d80f1453..6631d0677e 100644--- a/backend/plugin/schema/mysql/generate_migration_test.go+++ b/backend/plugin/schema/mysql/generate_migration_test.go@@ -236,3 +236,48 @@ func TestDiffMigrationPartitionAliasDoesNotMaskTableExistence(t *testing.T) {a.Equal(schema.MetadataDiffActionCreate, tableAction(diff, "archive"))})}++// A MariaDB table that loses WITH SYSTEM VERSIONING silently stops keeping history, and+// explicit ROW START/ROW END columns are rejected (4125) without both that option and a+// PERIOD FOR SYSTEM_TIME element.+func TestDiffMigrationKeepsSystemVersioning(t *testing.T) {+ a := require.New(t)+ implicit := &metadatapb.TableMetadata{+ Name: "history",+ Engine: "InnoDB",+ CreateOptions: "WITH SYSTEM VERSIONING",+ Columns: []*metadatapb.ColumnMetadata{{Name: "id", Type: "int", Nullable: false}},+ Indexes: []*metadatapb.IndexMetadata{+ {Name: "PRIMARY", Expressions: []string{"id"}, Primary: true, Unique: true, Visible: true, Type: "BTREE"},+ },+ }+ explicit := &metadatapb.TableMetadata{+ Name: "explicit_period",+ Engine: "InnoDB",+ CreateOptions: "WITH SYSTEM VERSIONING",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int", Nullable: true},+ {Name: "valid_from", Type: "timestamp(6)", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW START"}},+ {Name: "valid_to", Type: "timestamp(6)", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW END"}},+ },+ }++ migration, err := schema.DiffMigration(storepb.Engine_MYSQL, databaseOf(), databaseOf(implicit, explicit))+ a.NoError(err)+ a.Equal("CREATE TABLE IF NOT EXISTS `explicit_period` (\n"++ " `id` int,\n"++ " `valid_from` timestamp(6) GENERATED ALWAYS AS ROW START NOT NULL,\n"++ " `valid_to` timestamp(6) GENERATED ALWAYS AS ROW END NOT NULL,\n"++ " PERIOD FOR SYSTEM_TIME (`valid_from`, `valid_to`)\n"++ ") ENGINE=InnoDB WITH SYSTEM VERSIONING;\n"++ "\n"++ "CREATE TABLE IF NOT EXISTS `history` (\n"++ " `id` int NOT NULL,\n"++ " PRIMARY KEY (`id`)\n"++ ") ENGINE=InnoDB WITH SYSTEM VERSIONING;\n"++ "\n", migration)++ migration, err = schema.DiffMigration(storepb.Engine_MYSQL, databaseOf(implicit, explicit), databaseOf(implicit, explicit))+ a.NoError(err)+ a.Empty(migration)+}diff --git a/backend/plugin/schema/mysql/get_database_definition.go b/backend/plugin/schema/mysql/get_database_definition.goindex 5338aef90b..42a91df95f 100644--- a/backend/plugin/schema/mysql/get_database_definition.go+++ b/backend/plugin/schema/mysql/get_database_definition.go@@ -556,6 +556,7 @@ func writeTable(out *strings.Builder, table *metadatapb.TableMetadata) error {return err}}+ writeSystemTimePeriod(out, table)if err := printPrimaryKeyClause(out, table.Indexes); err != nil {return err@@ -609,6 +610,8 @@ func writeTable(out *strings.Builder, table *metadatapb.TableMetadata) error {}}+ writeSystemVersioningOption(out, table)+if len(table.Partitions) > 0 {if err := printPartitionClause(out, table.Partitions, table.Engine); err != nil {return err@@ -1214,7 +1217,9 @@ func printColumnClause(buf *strings.Builder, column *metadatapb.ColumnMetadata,}}- if column.Generation != nil && column.Generation.Expression != "" {+ if isSystemTimeColumn(column) {+ writeSystemTimeColumnAttribute(buf, column)+ } else if column.Generation != nil && column.Generation.Expression != "" {if _, err := fmt.Fprintf(buf, " GENERATED ALWAYS AS (%s) ", column.Generation.Expression); err != nil {return err}@@ -1710,6 +1715,7 @@ func writeTableSDL(buf *strings.Builder, table *metadatapb.TableMetadata) errorreturn err}}+ writeSystemTimePeriod(buf, table)if err := printPrimaryKeyClause(buf, table.Indexes); err != nil {return err@@ -1782,6 +1788,7 @@ func writeTableSDL(buf *strings.Builder, table *metadatapb.TableMetadata) errorreturn err}}+ writeSystemVersioningOption(buf, table)// Partitioning is a table-level clause that follows the table options. The omni// loader accepts the version-gated /*!50100 PARTITION BY … */ executable comment@@ -1812,7 +1819,8 @@ func filterCreateOptions(createOptions string) string {return ""}kept := make([]string, 0, 4)- for _, tok := range strings.Fields(createOptions) {+ // The system-versioning marker is written separately by writeSystemVersioningOption.+ for _, tok := range strings.Fields(strings.Replace(createOptions, systemVersioningOption, "", 1)) {if strings.EqualFold(tok, "partitioned") {continue}diff --git a/backend/plugin/schema/mysql/get_database_definition_test.go b/backend/plugin/schema/mysql/get_database_definition_test.goindex 4f16e7f115..410292aeb7 100644--- a/backend/plugin/schema/mysql/get_database_definition_test.go+++ b/backend/plugin/schema/mysql/get_database_definition_test.go@@ -52,6 +52,41 @@ func TestGetTableDefinitionWithCheckConstraint(t *testing.T) {require.NotContains(t, got, `\'`)}+func TestGetTableDefinitionWithSystemVersioning(t *testing.T) {+ table := &metadatapb.TableMetadata{+ Name: "explicit_period",+ Columns: []*metadatapb.ColumnMetadata{+ {Name: "id", Type: "int", Nullable: true, Default: "NULL"},+ {Name: "valid_from", Type: "timestamp(6)", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW START"}},+ {Name: "valid_to", Type: "timestamp(6)", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW END"}},+ },+ Engine: "InnoDB",+ Charset: "utf8mb4",+ Collation: "utf8mb4_general_ci",+ Comment: "versioned",+ CreateOptions: "WITH SYSTEM VERSIONING",+ }++ got, err := GetTableDefinition("", table, nil)+ require.NoError(t, err)++ want := "--\n" ++ "-- Table structure for `explicit_period`\n" ++ "--\n" ++ "CREATE TABLE `explicit_period` (\n" ++ " `id` int DEFAULT NULL,\n" ++ " `valid_from` timestamp(6) GENERATED ALWAYS AS ROW START NOT NULL,\n" ++ " `valid_to` timestamp(6) GENERATED ALWAYS AS ROW END NOT NULL,\n" ++ " PERIOD FOR SYSTEM_TIME (`valid_from`, `valid_to`)\n" ++ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='versioned' WITH SYSTEM VERSIONING;\n\n"+ require.Equal(t, want, got)++ // The schema-text diff path loads a dump with the omni MySQL loader. Loading the+ // table unversioned would let the diff recreate it without history, so it must fail.+ _, err = GetDatabaseMetadata(got)+ require.Error(t, err)+}+func TestGetFunctionDefinitionWithMultilineParameters(t *testing.T) {function := &metadatapb.FunctionMetadata{Name: "f1",diff --git a/backend/plugin/schema/mysql/system_versioning.go b/backend/plugin/schema/mysql/system_versioning.gonew file mode 100644index 0000000000..01c56809fc--- /dev/null+++ b/backend/plugin/schema/mysql/system_versioning.go@@ -0,0 +1,64 @@+package mysql++import (+ "strings"++ metadatapb "github.com/bytebase/omni/metadata"+)++// MariaDB system versioning is written as plain MariaDB syntax, not inside a+// /*M! */ comment: the omni MySQL loader that parses schema text for diffs would+// skip the comment and load the table unversioned, so a diff would silently+// recreate it without history. It rejects the plain syntax instead.++// systemVersioningOption is what the MariaDB sync appends to+// TableMetadata.CreateOptions for a system-versioned table. information_schema+// reports versioning only as a TABLE_TYPE, which TableMetadata has no field for.+const systemVersioningOption = "WITH SYSTEM VERSIONING"++func isSystemVersioned(table *metadatapb.TableMetadata) bool {+ return strings.Contains(table.CreateOptions, systemVersioningOption)+}++func writeSystemVersioningOption(buf *strings.Builder, table *metadatapb.TableMetadata) {+ if isSystemVersioned(table) {+ _, _ = buf.WriteString(" " + systemVersioningOption)+ }+}++// isSystemTimeColumn reports a MariaDB column declared GENERATED ALWAYS AS ROW START+// or ROW END, which information_schema reports as a stored generated column with that+// text as its expression.+func isSystemTimeColumn(column *metadatapb.ColumnMetadata) bool {+ generation := column.GetGeneration()+ if generation == nil || generation.Type != metadatapb.GenerationMetadata_TYPE_STORED {+ return false+ }+ return generation.Expression == "ROW START" || generation.Expression == "ROW END"+}++// writeSystemTimeColumnAttribute writes the attribute without the parentheses and+// storage kind of an ordinary generated column, which MariaDB rejects here.+func writeSystemTimeColumnAttribute(buf *strings.Builder, column *metadatapb.ColumnMetadata) {+ _, _ = buf.WriteString(" GENERATED ALWAYS AS " + column.Generation.Expression)+}++// writeSystemTimePeriod writes the PERIOD FOR SYSTEM_TIME table element that MariaDB+// requires alongside explicit ROW START and ROW END columns.+func writeSystemTimePeriod(buf *strings.Builder, table *metadatapb.TableMetadata) {+ var start, end string+ for _, column := range table.Columns {+ if !isSystemTimeColumn(column) {+ continue+ }+ if column.Generation.Expression == "ROW START" {+ start = column.Name+ } else {+ end = column.Name+ }+ }+ if start == "" || end == "" {+ return+ }+ _, _ = buf.WriteString(",\n PERIOD FOR SYSTEM_TIME (`" + start + "`, `" + end + "`)")+}