Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(mariadb): sync sequences and system-versioned tables (#21665)

rebelicea72a8b87c48e
diff --git a/backend/common/testcontainer/shared.go b/backend/common/testcontainer/shared.go
index 8be82c1ab6..d9910e420d 100644
--- a/backend/common/testcontainer/shared.go
+++ b/backend/common/testcontainer/shared.go
@@ -23,6 +23,9 @@ var (
sharedMSSQL = startOnce(getMSSQLContainer)
sharedTiDB = startOnce(getTiDBContainer)
sharedMongo = startOnce(getMongoDBContainer)
+ sharedMariaDB = startOnce(mariaDBStarter("mariadb:11.8.8"))
+ // 10.7 is the oldest MariaDB release Bytebase supports.
+ sharedMariaDB107 = startOnce(mariaDBStarter("mariadb:10.7.8"))
startedMu sync.Mutex
started []*Container
@@ -103,6 +106,14 @@ func SharedTiDBContainer(t testing.TB) *Container { return shared(t, sharedTiDB)
// each with NewMongoDatabase.
func SharedMongoDBContainer(t testing.TB) *Container { return shared(t, sharedMongo) }
+// SharedMariaDBContainer is SharedPgContainer for MariaDB 11.8; tests take a
+// database each with NewMariaDBDatabase.
+func SharedMariaDBContainer(t testing.TB) *Container { return shared(t, sharedMariaDB) }
+
+// SharedMariaDB107Container is SharedMariaDBContainer for MariaDB 10.7, the
+// oldest supported release; tests take a database each with NewMariaDB107Database.
+func SharedMariaDB107Container(t testing.TB) *Container { return shared(t, sharedMariaDB107) }
+
// NewPgDatabase creates an empty database for the test on the shared
// PostgreSQL container and returns its name with a superuser handle to it.
// Anything cluster-wide the test creates, such as a role, should be named
@@ -139,7 +150,24 @@ func createPgDatabase(t testing.TB, container *Container) (string, *sql.DB) {
// container and returns the container with the database's name.
func NewTiDBDatabase(t testing.TB) (*Container, string) {
t.Helper()
- container := SharedTiDBContainer(t)
+ return createMySQLProtocolDatabase(t, SharedTiDBContainer(t))
+}
+
+// NewMariaDBDatabase creates an empty database for the test on the shared
+// MariaDB container and returns the container with the database's name.
+func NewMariaDBDatabase(t testing.TB) (*Container, string) {
+ t.Helper()
+ return createMySQLProtocolDatabase(t, SharedMariaDBContainer(t))
+}
+
+// NewMariaDB107Database is NewMariaDBDatabase on the shared MariaDB 10.7 container.
+func NewMariaDB107Database(t testing.TB) (*Container, string) {
+ t.Helper()
+ return createMySQLProtocolDatabase(t, SharedMariaDB107Container(t))
+}
+
+func createMySQLProtocolDatabase(t testing.TB, container *Container) (*Container, string) {
+ t.Helper()
name := newDatabaseName()
_, err := container.GetDB().Exec("CREATE DATABASE " + name)
require.NoError(t, err)
diff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.go
index 60432aecf4..8bace1dbf8 100644
--- a/backend/common/testcontainer/testcontainer.go
+++ b/backend/common/testcontainer/testcontainer.go
@@ -243,6 +243,26 @@ func GetMySQLContainer(ctx context.Context) (*Container, error) {
})
}
+// mariaDBStarter starts the MariaDB image, whose root password is root-password.
+func mariaDBStarter(image string) func(context.Context) (*Container, error) {
+ return func(ctx context.Context) (*Container, error) {
+ const user, password = "root", "root-password"
+ return start(ctx, testcontainers.ContainerRequest{
+ Image: image,
+ Env: map[string]string{"MARIADB_ROOT_PASSWORD": password},
+ WaitingFor: wait.ForLog("ready for connections").WithOccurrence(2).WithStartupTimeout(5 * time.Minute),
+ }, spec{
+ port: "3306/tcp",
+ username: user,
+ password: password,
+ driver: "mysql",
+ dsn: func(c *Container) string {
+ return fmt.Sprintf("%s:%s@tcp(%s:%s)/?multiStatements=true", c.username, c.password, c.host, c.port)
+ },
+ })
+ }
+}
+
// mssqlTLSConf points SQL Server at the certificate getMSSQLContainer copies in,
// leaving encryption up to the client.
const mssqlTLSConf = `[network]
diff --git a/backend/plugin/db/mysql/main_test.go b/backend/plugin/db/mysql/main_test.go
new file mode 100644
index 0000000000..0a9c4993be
--- /dev/null
+++ b/backend/plugin/db/mysql/main_test.go
@@ -0,0 +1,9 @@
+package mysql
+
+import (
+ "testing"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+)
+
+func TestMain(m *testing.M) { testcontainer.Main(m) }
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.go
index 58a2d876de..7bcd0f82d9 100644
--- a/backend/plugin/db/mysql/mysql.go
+++ b/backend/plugin/db/mysql/mysql.go
@@ -40,6 +40,9 @@ import (
var (
baseTableType = "BASE TABLE"
viewTableType = "VIEW"
+ // MariaDB-only TABLE_TYPE values. Both kinds share the table namespace.
+ mariaDBSystemVersionedTableType = "SYSTEM VERSIONED"
+ mariaDBSequenceTableType = "SEQUENCE"
_ db.Driver = (*Driver)(nil)
)
diff --git a/backend/plugin/db/mysql/sync.go b/backend/plugin/db/mysql/sync.go
index 6fbef7422c..9985ed4724 100644
--- a/backend/plugin/db/mysql/sync.go
+++ b/backend/plugin/db/mysql/sync.go
@@ -583,6 +583,10 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
return nil, util.FormatErrorWithQuery(err, tableQuery)
}
defer tableRows.Close()
+ // Keyed on the registered engine, not the server: an instance registered as MySQL
+ // keeps its MySQL sync result even when the server is MariaDB.
+ mariaDBInstance := d.dbType == storepb.Engine_MARIADB
+ var sequences []*metadatapb.SequenceMetadata
for tableRows.Next() {
var tableName, tableType, engine, collation, createOptions, comment, charset string
var rowCount, dataSize, indexSize, dataFree int64
@@ -612,8 +616,11 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
// The partition details are fetched separately by listPartitionTables.
key := db.TableKey{Schema: "", Table: tableName}
- switch tableType {
- case baseTableType:
+ // A MariaDB system-versioned table is synced as a table: its hidden row_start
+ // and row_end columns are absent from information_schema.COLUMNS, and explicit
+ // ones arrive as stored generated columns whose expression is ROW START/ROW END.
+ switch {
+ case tableType == baseTableType, mariaDBInstance && tableType == mariaDBSystemVersionedTableType:
columns := columnMap[key]
tableMetadata := &metadatapb.TableMetadata{
Name: tableName,
@@ -635,6 +642,11 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
if tableCollation.Valid {
tableMetadata.Collation = tableCollation.String
}
+ if tableType == mariaDBSystemVersionedTableType {
+ // Recorded as the table option SHOW CREATE TABLE prints; the DDL writers in
+ // backend/plugin/schema/mysql look for it to keep the versioning.
+ tableMetadata.CreateOptions = strings.TrimSpace(createOptions + " WITH SYSTEM VERSIONING")
+ }
var indexNames []string
if indexes, ok := indexMap[key]; ok {
for indexName := range indexes {
@@ -647,10 +659,16 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
}
schemaMetadata.Tables = append(schemaMetadata.Tables, tableMetadata)
- case viewTableType:
+ case tableType == viewTableType:
if view, ok := viewMap[key]; ok {
schemaMetadata.Views = append(schemaMetadata.Views, view)
}
+ case mariaDBInstance && tableType == mariaDBSequenceTableType:
+ sequences = append(sequences, &metadatapb.SequenceMetadata{
+ Name: tableName,
+ DataType: mariaDBSequenceDataType(columnMap[key]),
+ Comment: comment,
+ })
default:
// Unknown table type, skip
}
@@ -658,6 +676,10 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
if err := tableRows.Err(); err != nil {
return nil, util.FormatErrorWithQuery(err, tableQuery)
}
+ if err := d.readMariaDBSequenceAttributes(ctx, d.databaseName, sequences); err != nil {
+ return nil, err
+ }
+ schemaMetadata.Sequences = sequences
databaseMetadata := &metadatapb.DatabaseSchemaMetadata{
Name: d.databaseName,
@@ -683,6 +705,49 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
return databaseMetadata, err
}
+var displayWidthRegexp = regexp.MustCompile(`\(\d+\)`)
+
+// mariaDBSequenceDataType derives a sequence's AS type from its start_value column:
+// with the display width removed, the column type spells the type the way
+// information_schema.SEQUENCES.DATA_TYPE does (MariaDB 11.5+), e.g. "int unsigned".
+func mariaDBSequenceDataType(columns []*metadatapb.ColumnMetadata) string {
+ for _, column := range columns {
+ if column.Name == "start_value" {
+ return displayWidthRegexp.ReplaceAllString(column.Type, "")
+ }
+ }
+ return ""
+}
+
+// readMariaDBSequenceAttributes reads each sequence's own row. information_schema.SEQUENCES
+// is no substitute: it exists only from MariaDB 11.5 and has no cache size. A sequence whose
+// row cannot be read, such as one the user has no SELECT privilege on, keeps only its name.
+func (d *Driver) readMariaDBSequenceAttributes(ctx context.Context, databaseName string, sequences []*metadatapb.SequenceMetadata) error {
+ for _, sequence := range sequences {
+ query := fmt.Sprintf(
+ "SELECT start_value, minimum_value, maximum_value, increment, cache_size, cycle_option FROM `%s`.`%s`",
+ strings.ReplaceAll(databaseName, "`", "``"),
+ strings.ReplaceAll(sequence.Name, "`", "``"),
+ )
+ var start, minValue, maxValue, increment, cacheSize string
+ var cycle bool
+ if err := d.db.QueryRowContext(ctx, query).Scan(&start, &minValue, &maxValue, &increment, &cacheSize, &cycle); err != nil {
+ if ctx.Err() != nil {
+ return ctx.Err()
+ }
+ slog.Warn("failed to read MariaDB sequence attributes", slog.String("database", databaseName), slog.String("sequence", sequence.Name), log.BBError(err))
+ continue
+ }
+ sequence.Start = start
+ sequence.MinValue = minValue
+ sequence.MaxValue = maxValue
+ sequence.Increment = increment
+ sequence.CacheSize = cacheSize
+ sequence.Cycle = cycle
+ }
+ return nil
+}
+
func (d *Driver) getEventList(ctx context.Context, databaseName string) ([]*metadatapb.EventMetadata, error) {
listEventsQuery := `
SELECT
diff --git a/backend/plugin/db/mysql/sync_test.go b/backend/plugin/db/mysql/sync_test.go
index 969d71f0d3..e5c89421bf 100644
--- a/backend/plugin/db/mysql/sync_test.go
+++ b/backend/plugin/db/mysql/sync_test.go
@@ -1,12 +1,21 @@
package mysql
import (
+ "context"
"database/sql"
+ "fmt"
+ "maps"
+ "slices"
"strings"
"testing"
metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/google/go-cmp/cmp"
"github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/testing/protocmp"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
func TestSetColumnMetadataDefault(t *testing.T) {
@@ -902,3 +911,130 @@ func TestStripReturnsCharset(t *testing.T) {
require.Equal(t, tc.want, stripReturnsCharset(tc.def), tc.name)
}
}
+
+// TestSyncMariaDBTableNamespace covers the MariaDB objects that share the table
+// namespace with base tables. A consumer can only conclude that a table name is
+// free if sync lists every one of them.
+func TestSyncMariaDBTableNamespace(t *testing.T) {
+ t.Parallel()
+ for _, tc := range []struct {
+ version string
+ newDatabase func(testing.TB) (*testcontainer.Container, string)
+ }{
+ {version: "11.8", newDatabase: testcontainer.NewMariaDBDatabase},
+ {version: "10.7", newDatabase: testcontainer.NewMariaDB107Database},
+ } {
+ t.Run(tc.version, func(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+ container, database := tc.newDatabase(t)
+ driver := openTestDriver(ctx, t, storepb.Engine_MARIADB, container, container.GetUsername(), container.GetPassword(), database)
+ _, err := driver.db.ExecContext(ctx, `
+CREATE TABLE parent (id INT PRIMARY KEY);
+CREATE TABLE history (
+ id INT PRIMARY KEY,
+ amount INT,
+ parent_id INT,
+ KEY idx_amount (amount),
+ CONSTRAINT fk_parent FOREIGN KEY (parent_id) REFERENCES parent (id),
+ CONSTRAINT chk_amount CHECK (amount > 0)
+) WITH SYSTEM VERSIONING COMMENT 'versioned';
+CREATE TRIGGER trg_history BEFORE INSERT ON history FOR EACH ROW SET NEW.amount = NEW.amount;
+CREATE TABLE explicit_period (
+ id INT,
+ valid_from TIMESTAMP(6) GENERATED ALWAYS AS ROW START,
+ valid_to TIMESTAMP(6) GENERATED ALWAYS AS ROW END,
+ PERIOD FOR SYSTEM_TIME (valid_from, valid_to)
+) WITH SYSTEM VERSIONING;
+CREATE SEQUENCE seq_default;
+CREATE SEQUENCE seq_custom START WITH 100 INCREMENT BY 5 MINVALUE 10 MAXVALUE 1000 CACHE 20 CYCLE COMMENT 'sequence comment';
+`)
+ require.NoError(t, err)
+
+ metadata, err := driver.SyncDBSchema(ctx)
+ require.NoError(t, err)
+ require.Len(t, metadata.Schemas, 1)
+ schema := metadata.Schemas[0]
+
+ tables := make(map[string]*metadatapb.TableMetadata)
+ for _, table := range schema.Tables {
+ tables[table.Name] = table
+ }
+ require.ElementsMatch(t, []string{"explicit_period", "history", "parent"}, slices.Collect(maps.Keys(tables)))
+
+ history := tables["history"]
+ require.Equal(t, []string{"id", "amount", "parent_id"}, columnNames(history.Columns))
+ var indexNames []string
+ for _, index := range history.Indexes {
+ indexNames = append(indexNames, index.Name)
+ }
+ require.Equal(t, []string{"PRIMARY", "fk_parent", "idx_amount"}, indexNames)
+ require.Equal(t, []string{"id"}, history.Indexes[0].Expressions)
+ require.Len(t, history.ForeignKeys, 1)
+ require.Equal(t, "fk_parent", history.ForeignKeys[0].Name)
+ require.Equal(t, "parent", history.ForeignKeys[0].ReferencedTable)
+ require.Empty(t, cmp.Diff([]*metadatapb.CheckConstraintMetadata{{Name: "chk_amount", Expression: "`amount` > 0"}}, history.CheckConstraints, protocmp.Transform()))
+ require.Len(t, history.Triggers, 1)
+ require.Equal(t, "trg_history", history.Triggers[0].Name)
+ require.Equal(t, "versioned", history.Comment)
+ require.Equal(t, "WITH SYSTEM VERSIONING", history.CreateOptions)
+ require.Empty(t, tables["parent"].CreateOptions)
+
+ explicitPeriod := tables["explicit_period"]
+ require.Equal(t, []string{"id", "valid_from", "valid_to"}, columnNames(explicitPeriod.Columns))
+ require.Equal(t, "WITH SYSTEM VERSIONING", explicitPeriod.CreateOptions)
+ require.Empty(t, cmp.Diff(&metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW START"}, explicitPeriod.Columns[1].Generation, protocmp.Transform()))
+ require.Empty(t, cmp.Diff(&metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW END"}, explicitPeriod.Columns[2].Generation, protocmp.Transform()))
+
+ require.Empty(t, cmp.Diff([]*metadatapb.SequenceMetadata{
+ {Name: "seq_custom", DataType: "bigint", Start: "100", MinValue: "10", MaxValue: "1000", Increment: "5", CacheSize: "20", Cycle: true, Comment: "sequence comment"},
+ {Name: "seq_default", DataType: "bigint", Start: "1", MinValue: "1", MaxValue: "9223372036854775806", Increment: "1", CacheSize: "1000"},
+ }, schema.Sequences, protocmp.Transform()))
+
+ // The same server registered as a MySQL instance keeps the MySQL sync result.
+ asMySQL := openTestDriver(ctx, t, storepb.Engine_MYSQL, container, container.GetUsername(), container.GetPassword(), database)
+ metadata, err = asMySQL.SyncDBSchema(ctx)
+ require.NoError(t, err)
+ require.Len(t, metadata.Schemas, 1)
+ require.Len(t, metadata.Schemas[0].Tables, 1)
+ require.Equal(t, "parent", metadata.Schemas[0].Tables[0].Name)
+ require.Empty(t, metadata.Schemas[0].Sequences)
+ })
+ }
+}
+
+func TestSyncMariaDBSequenceAttributes(t *testing.T) {
+ t.Parallel()
+ ctx := context.Background()
+ container, database := testcontainer.NewMariaDBDatabase(t)
+ reader := database + "_reader"
+ _, err := container.GetDB().ExecContext(ctx, fmt.Sprintf(`
+CREATE SEQUENCE %[1]s.seq_typed AS INT UNSIGNED START WITH 7 MAXVALUE 4000000000;
+CREATE SEQUENCE %[1]s.seq_unreadable;
+CREATE USER '%[2]s'@'%%' IDENTIFIED BY 'reader-password';
+GRANT SELECT ON %[1]s.seq_typed TO '%[2]s'@'%%';
+GRANT INSERT ON %[1]s.seq_unreadable TO '%[2]s'@'%%';
+`, database, reader))
+ require.NoError(t, err)
+
+ driver := openTestDriver(ctx, t, storepb.Engine_MARIADB, container, reader, "reader-password", database)
+ metadata, err := driver.SyncDBSchema(ctx)
+ require.NoError(t, err)
+ require.Len(t, metadata.Schemas, 1)
+ schema := metadata.Schemas[0]
+
+ require.Empty(t, schema.Tables)
+ // Without SELECT on a sequence its row is unreadable, but the name is still taken.
+ require.Empty(t, cmp.Diff([]*metadatapb.SequenceMetadata{
+ {Name: "seq_typed", DataType: "int unsigned", Start: "7", MinValue: "1", MaxValue: "4000000000", Increment: "1", CacheSize: "1000"},
+ {Name: "seq_unreadable", DataType: "bigint"},
+ }, schema.Sequences, protocmp.Transform()))
+}
+
+func columnNames(columns []*metadatapb.ColumnMetadata) []string {
+ var names []string
+ for _, column := range columns {
+ names = append(names, column.Name)
+ }
+ return names
+}
diff --git a/backend/plugin/db/mysql/testutil_test.go b/backend/plugin/db/mysql/testutil_test.go
new file mode 100644
index 0000000000..37d985ca98
--- /dev/null
+++ b/backend/plugin/db/mysql/testutil_test.go
@@ -0,0 +1,33 @@
+package mysql
+
+import (
+ "context"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+)
+
+func openTestDriver(ctx context.Context, t *testing.T, engine storepb.Engine, container *testcontainer.Container, username, password, database string) *Driver {
+ t.Helper()
+
+ d, err := newDriver().Open(ctx, engine, db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Type: storepb.DataSourceType_ADMIN,
+ Username: username,
+ Host: container.GetHost(),
+ Port: container.GetPort(),
+ },
+ Password: password,
+ ConnectionContext: db.ConnectionContext{DatabaseName: database},
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, d.Close(ctx)) })
+
+ driver, ok := d.(*Driver)
+ require.True(t, ok)
+ return driver
+}
diff --git a/backend/plugin/schema/mysql/generate_migration.go b/backend/plugin/schema/mysql/generate_migration.go
index 1a2acfb895..063955daaf 100644
--- a/backend/plugin/schema/mysql/generate_migration.go
+++ b/backend/plugin/schema/mysql/generate_migration.go
@@ -546,6 +546,7 @@ func writeCreateTableWithoutForeignKeys(buf *strings.Builder, tableName string,
// generated-column ordering: GENERATED ALWAYS AS (...) precedes NOT NULL/SRID).
writeColumnDefinitionBody(buf, col)
}
+ writeSystemTimePeriod(buf, table)
// Write primary key constraint inline if exists
for _, index := range table.Indexes {
@@ -611,6 +612,7 @@ func writeCreateTableWithoutForeignKeys(buf *strings.Builder, tableName string,
_, _ = buf.WriteString(table.Comment)
_, _ = buf.WriteString("'")
}
+ writeSystemVersioningOption(buf, table)
// Partitioning is part of the table definition, so it has to be emitted here: MySQL
// cannot partition a table after the fact without rewriting it. printPartitionClause
@@ -708,17 +710,21 @@ func writeColumnDefinitionBody(buf *strings.Builder, column *metadatapb.ColumnMe
// and generated columns. Generated columns never carry DEFAULT / AUTO_INCREMENT / ON
// UPDATE, so those clauses are intentionally omitted.
func writeGeneratedColumnAttributes(buf *strings.Builder, column *metadatapb.ColumnMetadata) {
- _, _ = buf.WriteString(" GENERATED ALWAYS AS (")
- _, _ = buf.WriteString(column.Generation.Expression)
- _, _ = buf.WriteString(") ")
- switch column.Generation.Type {
- case metadatapb.GenerationMetadata_TYPE_STORED:
- _, _ = buf.WriteString("STORED")
- case metadatapb.GenerationMetadata_TYPE_VIRTUAL:
- _, _ = buf.WriteString("VIRTUAL")
- default:
- // Default to VIRTUAL for unknown types
- _, _ = buf.WriteString("VIRTUAL")
+ if isSystemTimeColumn(column) {
+ writeSystemTimeColumnAttribute(buf, column)
+ } else {
+ _, _ = buf.WriteString(" GENERATED ALWAYS AS (")
+ _, _ = buf.WriteString(column.Generation.Expression)
+ _, _ = buf.WriteString(") ")
+ switch column.Generation.Type {
+ case metadatapb.GenerationMetadata_TYPE_STORED:
+ _, _ = buf.WriteString("STORED")
+ case metadatapb.GenerationMetadata_TYPE_VIRTUAL:
+ _, _ = buf.WriteString("VIRTUAL")
+ default:
+ // Default to VIRTUAL for unknown types
+ _, _ = buf.WriteString("VIRTUAL")
+ }
}
if !column.Nullable {
diff --git a/backend/plugin/schema/mysql/generate_migration_test.go b/backend/plugin/schema/mysql/generate_migration_test.go
index 88d80f1453..6631d0677e 100644
--- a/backend/plugin/schema/mysql/generate_migration_test.go
+++ b/backend/plugin/schema/mysql/generate_migration_test.go
@@ -236,3 +236,48 @@ func TestDiffMigrationPartitionAliasDoesNotMaskTableExistence(t *testing.T) {
a.Equal(schema.MetadataDiffActionCreate, tableAction(diff, "archive"))
})
}
+
+// A MariaDB table that loses WITH SYSTEM VERSIONING silently stops keeping history, and
+// explicit ROW START/ROW END columns are rejected (4125) without both that option and a
+// PERIOD FOR SYSTEM_TIME element.
+func TestDiffMigrationKeepsSystemVersioning(t *testing.T) {
+ a := require.New(t)
+ implicit := &metadatapb.TableMetadata{
+ Name: "history",
+ Engine: "InnoDB",
+ CreateOptions: "WITH SYSTEM VERSIONING",
+ Columns: []*metadatapb.ColumnMetadata{{Name: "id", Type: "int", Nullable: false}},
+ Indexes: []*metadatapb.IndexMetadata{
+ {Name: "PRIMARY", Expressions: []string{"id"}, Primary: true, Unique: true, Visible: true, Type: "BTREE"},
+ },
+ }
+ explicit := &metadatapb.TableMetadata{
+ Name: "explicit_period",
+ Engine: "InnoDB",
+ CreateOptions: "WITH SYSTEM VERSIONING",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int", Nullable: true},
+ {Name: "valid_from", Type: "timestamp(6)", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW START"}},
+ {Name: "valid_to", Type: "timestamp(6)", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW END"}},
+ },
+ }
+
+ migration, err := schema.DiffMigration(storepb.Engine_MYSQL, databaseOf(), databaseOf(implicit, explicit))
+ a.NoError(err)
+ a.Equal("CREATE TABLE IF NOT EXISTS `explicit_period` (\n"+
+ " `id` int,\n"+
+ " `valid_from` timestamp(6) GENERATED ALWAYS AS ROW START NOT NULL,\n"+
+ " `valid_to` timestamp(6) GENERATED ALWAYS AS ROW END NOT NULL,\n"+
+ " PERIOD FOR SYSTEM_TIME (`valid_from`, `valid_to`)\n"+
+ ") ENGINE=InnoDB WITH SYSTEM VERSIONING;\n"+
+ "\n"+
+ "CREATE TABLE IF NOT EXISTS `history` (\n"+
+ " `id` int NOT NULL,\n"+
+ " PRIMARY KEY (`id`)\n"+
+ ") ENGINE=InnoDB WITH SYSTEM VERSIONING;\n"+
+ "\n", migration)
+
+ migration, err = schema.DiffMigration(storepb.Engine_MYSQL, databaseOf(implicit, explicit), databaseOf(implicit, explicit))
+ a.NoError(err)
+ a.Empty(migration)
+}
diff --git a/backend/plugin/schema/mysql/get_database_definition.go b/backend/plugin/schema/mysql/get_database_definition.go
index 5338aef90b..42a91df95f 100644
--- a/backend/plugin/schema/mysql/get_database_definition.go
+++ b/backend/plugin/schema/mysql/get_database_definition.go
@@ -556,6 +556,7 @@ func writeTable(out *strings.Builder, table *metadatapb.TableMetadata) error {
return err
}
}
+ writeSystemTimePeriod(out, table)
if err := printPrimaryKeyClause(out, table.Indexes); err != nil {
return err
@@ -609,6 +610,8 @@ func writeTable(out *strings.Builder, table *metadatapb.TableMetadata) error {
}
}
+ writeSystemVersioningOption(out, table)
+
if len(table.Partitions) > 0 {
if err := printPartitionClause(out, table.Partitions, table.Engine); err != nil {
return err
@@ -1214,7 +1217,9 @@ func printColumnClause(buf *strings.Builder, column *metadatapb.ColumnMetadata,
}
}
- if column.Generation != nil && column.Generation.Expression != "" {
+ if isSystemTimeColumn(column) {
+ writeSystemTimeColumnAttribute(buf, column)
+ } else if column.Generation != nil && column.Generation.Expression != "" {
if _, err := fmt.Fprintf(buf, " GENERATED ALWAYS AS (%s) ", column.Generation.Expression); err != nil {
return err
}
@@ -1710,6 +1715,7 @@ func writeTableSDL(buf *strings.Builder, table *metadatapb.TableMetadata) error
return err
}
}
+ writeSystemTimePeriod(buf, table)
if err := printPrimaryKeyClause(buf, table.Indexes); err != nil {
return err
@@ -1782,6 +1788,7 @@ func writeTableSDL(buf *strings.Builder, table *metadatapb.TableMetadata) error
return err
}
}
+ writeSystemVersioningOption(buf, table)
// Partitioning is a table-level clause that follows the table options. The omni
// loader accepts the version-gated /*!50100 PARTITION BY … */ executable comment
@@ -1812,7 +1819,8 @@ func filterCreateOptions(createOptions string) string {
return ""
}
kept := make([]string, 0, 4)
- for _, tok := range strings.Fields(createOptions) {
+ // The system-versioning marker is written separately by writeSystemVersioningOption.
+ for _, tok := range strings.Fields(strings.Replace(createOptions, systemVersioningOption, "", 1)) {
if strings.EqualFold(tok, "partitioned") {
continue
}
diff --git a/backend/plugin/schema/mysql/get_database_definition_test.go b/backend/plugin/schema/mysql/get_database_definition_test.go
index 4f16e7f115..410292aeb7 100644
--- a/backend/plugin/schema/mysql/get_database_definition_test.go
+++ b/backend/plugin/schema/mysql/get_database_definition_test.go
@@ -52,6 +52,41 @@ func TestGetTableDefinitionWithCheckConstraint(t *testing.T) {
require.NotContains(t, got, `\'`)
}
+func TestGetTableDefinitionWithSystemVersioning(t *testing.T) {
+ table := &metadatapb.TableMetadata{
+ Name: "explicit_period",
+ Columns: []*metadatapb.ColumnMetadata{
+ {Name: "id", Type: "int", Nullable: true, Default: "NULL"},
+ {Name: "valid_from", Type: "timestamp(6)", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW START"}},
+ {Name: "valid_to", Type: "timestamp(6)", Generation: &metadatapb.GenerationMetadata{Type: metadatapb.GenerationMetadata_TYPE_STORED, Expression: "ROW END"}},
+ },
+ Engine: "InnoDB",
+ Charset: "utf8mb4",
+ Collation: "utf8mb4_general_ci",
+ Comment: "versioned",
+ CreateOptions: "WITH SYSTEM VERSIONING",
+ }
+
+ got, err := GetTableDefinition("", table, nil)
+ require.NoError(t, err)
+
+ want := "--\n" +
+ "-- Table structure for `explicit_period`\n" +
+ "--\n" +
+ "CREATE TABLE `explicit_period` (\n" +
+ " `id` int DEFAULT NULL,\n" +
+ " `valid_from` timestamp(6) GENERATED ALWAYS AS ROW START NOT NULL,\n" +
+ " `valid_to` timestamp(6) GENERATED ALWAYS AS ROW END NOT NULL,\n" +
+ " PERIOD FOR SYSTEM_TIME (`valid_from`, `valid_to`)\n" +
+ ") ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci COMMENT='versioned' WITH SYSTEM VERSIONING;\n\n"
+ require.Equal(t, want, got)
+
+ // The schema-text diff path loads a dump with the omni MySQL loader. Loading the
+ // table unversioned would let the diff recreate it without history, so it must fail.
+ _, err = GetDatabaseMetadata(got)
+ require.Error(t, err)
+}
+
func TestGetFunctionDefinitionWithMultilineParameters(t *testing.T) {
function := &metadatapb.FunctionMetadata{
Name: "f1",
diff --git a/backend/plugin/schema/mysql/system_versioning.go b/backend/plugin/schema/mysql/system_versioning.go
new file mode 100644
index 0000000000..01c56809fc
--- /dev/null
+++ b/backend/plugin/schema/mysql/system_versioning.go
@@ -0,0 +1,64 @@
+package mysql
+
+import (
+ "strings"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+)
+
+// MariaDB system versioning is written as plain MariaDB syntax, not inside a
+// /*M! */ comment: the omni MySQL loader that parses schema text for diffs would
+// skip the comment and load the table unversioned, so a diff would silently
+// recreate it without history. It rejects the plain syntax instead.
+
+// systemVersioningOption is what the MariaDB sync appends to
+// TableMetadata.CreateOptions for a system-versioned table. information_schema
+// reports versioning only as a TABLE_TYPE, which TableMetadata has no field for.
+const systemVersioningOption = "WITH SYSTEM VERSIONING"
+
+func isSystemVersioned(table *metadatapb.TableMetadata) bool {
+ return strings.Contains(table.CreateOptions, systemVersioningOption)
+}
+
+func writeSystemVersioningOption(buf *strings.Builder, table *metadatapb.TableMetadata) {
+ if isSystemVersioned(table) {
+ _, _ = buf.WriteString(" " + systemVersioningOption)
+ }
+}
+
+// isSystemTimeColumn reports a MariaDB column declared GENERATED ALWAYS AS ROW START
+// or ROW END, which information_schema reports as a stored generated column with that
+// text as its expression.
+func isSystemTimeColumn(column *metadatapb.ColumnMetadata) bool {
+ generation := column.GetGeneration()
+ if generation == nil || generation.Type != metadatapb.GenerationMetadata_TYPE_STORED {
+ return false
+ }
+ return generation.Expression == "ROW START" || generation.Expression == "ROW END"
+}
+
+// writeSystemTimeColumnAttribute writes the attribute without the parentheses and
+// storage kind of an ordinary generated column, which MariaDB rejects here.
+func writeSystemTimeColumnAttribute(buf *strings.Builder, column *metadatapb.ColumnMetadata) {
+ _, _ = buf.WriteString(" GENERATED ALWAYS AS " + column.Generation.Expression)
+}
+
+// writeSystemTimePeriod writes the PERIOD FOR SYSTEM_TIME table element that MariaDB
+// requires alongside explicit ROW START and ROW END columns.
+func writeSystemTimePeriod(buf *strings.Builder, table *metadatapb.TableMetadata) {
+ var start, end string
+ for _, column := range table.Columns {
+ if !isSystemTimeColumn(column) {
+ continue
+ }
+ if column.Generation.Expression == "ROW START" {
+ start = column.Name
+ } else {
+ end = column.Name
+ }
+ }
+ if start == "" || end == "" {
+ return
+ }
+ _, _ = buf.WriteString(",\n PERIOD FOR SYSTEM_TIME (`" + start + "`, `" + end + "`)")
+}