Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(oceanbase): sync external tables and materialized views (#21685)

rebelice43fbe0afdd5c
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.go
index 0d029352b7..8e984eab00 100644
--- a/backend/plugin/db/mysql/mysql.go
+++ b/backend/plugin/db/mysql/mysql.go
@@ -43,6 +43,8 @@ var (
// MariaDB-only TABLE_TYPE values. Both kinds share the table namespace.
mariaDBSystemVersionedTableType = "SYSTEM VERSIONED"
mariaDBSequenceTableType = "SEQUENCE"
+ // OceanBase-only TABLE_TYPE value; the kind shares the table namespace.
+ oceanBaseExternalTableType = "EXTERNAL TABLE"
_ db.Driver = (*Driver)(nil)
)
diff --git a/backend/plugin/db/mysql/sync.go b/backend/plugin/db/mysql/sync.go
index 9985ed4724..fcf1cdf392 100644
--- a/backend/plugin/db/mysql/sync.go
+++ b/backend/plugin/db/mysql/sync.go
@@ -17,6 +17,7 @@ import (
metadatapb "github.com/bytebase/omni/metadata"
"golang.org/x/text/encoding/charmap"
"golang.org/x/text/transform"
+ "google.golang.org/protobuf/proto"
"github.com/blang/semver/v4"
"github.com/pkg/errors"
@@ -586,6 +587,7 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
// Keyed on the registered engine, not the server: an instance registered as MySQL
// keeps its MySQL sync result even when the server is MariaDB.
mariaDBInstance := d.dbType == storepb.Engine_MARIADB
+ oceanBaseInstance := d.dbType == storepb.Engine_OCEANBASE
var sequences []*metadatapb.SequenceMetadata
for tableRows.Next() {
var tableName, tableType, engine, collation, createOptions, comment, charset string
@@ -669,6 +671,11 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
DataType: mariaDBSequenceDataType(columnMap[key]),
Comment: comment,
})
+ case oceanBaseInstance && tableType == oceanBaseExternalTableType:
+ schemaMetadata.ExternalTables = append(schemaMetadata.ExternalTables, &metadatapb.ExternalTableMetadata{
+ Name: tableName,
+ Columns: columnMap[key],
+ })
default:
// Unknown table type, skip
}
@@ -680,6 +687,13 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
return nil, err
}
schemaMetadata.Sequences = sequences
+ if oceanBaseInstance {
+ materializedViews, err := d.listOceanBaseMaterializedViews(ctx, d.databaseName, schemaMetadata.Tables)
+ if err != nil {
+ return nil, err
+ }
+ schemaMetadata.MaterializedViews = materializedViews
+ }
databaseMetadata := &metadatapb.DatabaseSchemaMetadata{
Name: d.databaseName,
@@ -748,6 +762,56 @@ func (d *Driver) readMariaDBSequenceAttributes(ctx context.Context, databaseName
return nil
}
+// listOceanBaseMaterializedViews reads the database's materialized views. OceanBase lists one
+// in neither information_schema.TABLES nor VIEWS, only its storage table __mv_container_<id>,
+// which carries the view's comment and indexes and stays synced as a table. A failed read, such
+// as one without SELECT on the oceanbase database, leaves the list empty; a caller can tell,
+// because a container table is then synced without its materialized view.
+func (d *Driver) listOceanBaseMaterializedViews(ctx context.Context, databaseName string, tables []*metadatapb.TableMetadata) ([]*metadatapb.MaterializedViewMetadata, error) {
+ query := `
+ SELECT MVIEW_NAME, CONTAINER_NAME, QUERY
+ FROM oceanbase.DBA_MVIEWS
+ WHERE OWNER = ?
+ ORDER BY MVIEW_NAME`
+ rows, err := d.db.QueryContext(ctx, query, databaseName)
+ if err != nil {
+ if ctx.Err() != nil {
+ return nil, ctx.Err()
+ }
+ slog.Warn("failed to list OceanBase materialized views", slog.String("database", databaseName), log.BBError(err))
+ return nil, nil
+ }
+ defer rows.Close()
+ containers := make(map[string]*metadatapb.TableMetadata, len(tables))
+ for _, table := range tables {
+ containers[table.Name] = table
+ }
+ var views []*metadatapb.MaterializedViewMetadata
+ for rows.Next() {
+ var name, containerName, definition string
+ if err := rows.Scan(&name, &containerName, &definition); err != nil {
+ return nil, err
+ }
+ view := &metadatapb.MaterializedViewMetadata{
+ Name: name,
+ Definition: definition,
+ // No MySQL-family DDL writer renders OceanBase's CREATE MATERIALIZED VIEW.
+ SkipDump: true,
+ }
+ if container, ok := containers[containerName]; ok {
+ view.Comment = container.Comment
+ for _, index := range container.Indexes {
+ view.Indexes = append(view.Indexes, proto.CloneOf(index))
+ }
+ }
+ views = append(views, view)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, util.FormatErrorWithQuery(err, query)
+ }
+ return views, nil
+}
+
func (d *Driver) getEventList(ctx context.Context, databaseName string) ([]*metadatapb.EventMetadata, error) {
listEventsQuery := `
SELECT
diff --git a/backend/plugin/db/mysql/sync_oceanbase_test.go b/backend/plugin/db/mysql/sync_oceanbase_test.go
new file mode 100644
index 0000000000..c0ec31fc07
--- /dev/null
+++ b/backend/plugin/db/mysql/sync_oceanbase_test.go
@@ -0,0 +1,291 @@
+package mysql
+
+import (
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "fmt"
+ "maps"
+ "net"
+ "os"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/go-sql-driver/mysql"
+ "github.com/google/go-cmp/cmp"
+ "github.com/stretchr/testify/require"
+ "google.golang.org/protobuf/testing/protocmp"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+)
+
+// oceanBaseDSNEnv names a MySQL-mode OceanBase tenant user that can create databases and users,
+// such as root@test:@tcp(127.0.0.1:2881)/. OceanBase needs about 8 GB, more than CI has; omni's
+// scripts/oceanbase-oracle.sh boots one with secure_file_priv set and exports the variable.
+const oceanBaseDSNEnv = "OCEANBASE_WALK_DSN"
+
+type oceanBaseTestServer struct {
+ host, port string
+ user string
+ password string
+ // tenant is the login suffix after "@" that every user of the tenant signs in with.
+ tenant string
+}
+
+func newOceanBaseTestServer(t *testing.T) oceanBaseTestServer {
+ t.Helper()
+ dsn := os.Getenv(oceanBaseDSNEnv)
+ if dsn == "" {
+ t.Skipf("%s is not set", oceanBaseDSNEnv)
+ }
+ cfg, err := mysql.ParseDSN(dsn)
+ require.NoError(t, err)
+ host, port, err := net.SplitHostPort(cfg.Addr)
+ require.NoError(t, err)
+ _, tenant, _ := strings.Cut(cfg.User, "@")
+ return oceanBaseTestServer{host: host, port: port, user: cfg.User, password: cfg.Passwd, tenant: tenant}
+}
+
+func (s oceanBaseTestServer) open(ctx context.Context, t *testing.T, engine storepb.Engine, user, password, database string) *Driver {
+ t.Helper()
+ d, err := newDriver().Open(ctx, engine, db.ConnectionConfig{
+ DataSource: &storepb.DataSource{
+ Type: storepb.DataSourceType_ADMIN,
+ Username: user,
+ Host: s.host,
+ Port: s.port,
+ },
+ Password: password,
+ ConnectionContext: db.ConnectionContext{DatabaseName: database},
+ })
+ require.NoError(t, err)
+ t.Cleanup(func() { require.NoError(t, d.Close(ctx)) })
+ driver, ok := d.(*Driver)
+ require.True(t, ok)
+ return driver
+}
+
+// TestSyncOceanBaseTableNamespace covers the OceanBase objects that share the table namespace
+// with base tables. A consumer can only conclude that a table name is free if sync lists every
+// one of them.
+func TestSyncOceanBaseTableNamespace(t *testing.T) {
+ server := newOceanBaseTestServer(t)
+ ctx := context.Background()
+ startNano := time.Now().UnixNano()
+ database := fmt.Sprintf("sync_namespace_%d", startNano)
+ reader := fmt.Sprintf("sync_reader_%d", startNano)
+
+ admin := server.open(ctx, t, storepb.Engine_OCEANBASE, server.user, server.password, "")
+ _, err := admin.db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE `%s`", database))
+ require.NoError(t, err)
+ t.Cleanup(func() {
+ _, err := admin.db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE `%s`", database))
+ require.NoError(t, err)
+ _, err = admin.db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS '%s'@'%%'", reader))
+ require.NoError(t, err)
+ })
+
+ driver := server.open(ctx, t, storepb.Engine_OCEANBASE, server.user, server.password, database)
+ for _, statement := range []string{
+ "CREATE TABLE t (id INT PRIMARY KEY, a INT) COMMENT 'base'",
+ "CREATE VIEW v AS SELECT id, a FROM t",
+ "CREATE EXTERNAL TABLE ext (c1 INT, c2 VARCHAR(10)) LOCATION = '/tmp' FORMAT = (TYPE = 'CSV')",
+ "CREATE MATERIALIZED VIEW mv COMMENT 'materialized' AS SELECT id, a FROM t",
+ "CREATE INDEX idx_mv_a ON mv (a)",
+ "CREATE MATERIALIZED VIEW LOG ON t WITH PRIMARY KEY (a) INCLUDING NEW VALUES",
+ "CREATE SEQUENCE seq",
+ } {
+ _, err := driver.db.ExecContext(ctx, statement)
+ require.NoError(t, err, statement)
+ }
+ var container string
+ require.NoError(t, driver.db.QueryRowContext(ctx,
+ "SELECT CONTAINER_NAME FROM oceanbase.DBA_MVIEWS WHERE OWNER = ? AND MVIEW_NAME = 'mv'", database,
+ ).Scan(&container))
+
+ census := oceanBaseObjectCensus(ctx, t, driver, database)
+ require.ElementsMatch(t, []string{"t", "v", "ext", "mv", "mlog$_t", container}, census)
+ for _, name := range census {
+ _, err := driver.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE `%s` (z INT)", name))
+ requireMySQLErrorNumber(t, err, 1050, name)
+ }
+ // A sequence has a namespace of its own.
+ _, err = driver.db.ExecContext(ctx, "CREATE TABLE seq (z INT)")
+ require.NoError(t, err)
+ _, err = driver.db.ExecContext(ctx, "DROP TABLE seq")
+ require.NoError(t, err)
+
+ metadata, err := driver.SyncDBSchema(ctx)
+ require.NoError(t, err)
+ require.Len(t, metadata.Schemas, 1)
+ schema := metadata.Schemas[0]
+ require.ElementsMatch(t, census, syncedRelationNames(schema))
+
+ tables := make(map[string]*metadatapb.TableMetadata)
+ for _, table := range schema.Tables {
+ tables[table.Name] = table
+ }
+ require.ElementsMatch(t, []string{"t", "mlog$_t", container}, slices.Collect(maps.Keys(tables)))
+ require.Equal(t, "materialized", tables[container].Comment)
+
+ require.Len(t, schema.ExternalTables, 1)
+ require.Equal(t, "ext", schema.ExternalTables[0].Name)
+ require.Equal(t, []string{"c1", "c2"}, columnNames(schema.ExternalTables[0].Columns))
+
+ require.Len(t, schema.MaterializedViews, 1)
+ mv := schema.MaterializedViews[0]
+ require.Equal(t, "mv", mv.Name)
+ require.Equal(t, fmt.Sprintf("select `%[1]s`.`t`.`id` AS `id`,`%[1]s`.`t`.`a` AS `a` from `%[1]s`.`t`", database), mv.Definition)
+ require.Equal(t, "materialized", mv.Comment)
+ require.True(t, mv.SkipDump)
+ require.Len(t, mv.Indexes, 1)
+ require.Equal(t, "idx_mv_a", mv.Indexes[0].Name)
+ require.Equal(t, []string{"a"}, mv.Indexes[0].Expressions)
+ require.Empty(t, cmp.Diff(tables[container].Indexes, mv.Indexes, protocmp.Transform()))
+
+ // Without SELECT on the oceanbase database the materialized views are unreadable; the
+ // container table, synced without its view, shows that the list is incomplete.
+ _, err = admin.db.ExecContext(ctx, fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY 'Reader_pw1'", reader))
+ require.NoError(t, err)
+ _, err = admin.db.ExecContext(ctx, fmt.Sprintf("GRANT SELECT, SHOW VIEW ON `%s`.* TO '%s'@'%%'", database, reader))
+ require.NoError(t, err)
+ asReader := server.open(ctx, t, storepb.Engine_OCEANBASE, reader+"@"+server.tenant, "Reader_pw1", database)
+ metadata, err = asReader.SyncDBSchema(ctx)
+ require.NoError(t, err)
+ require.Empty(t, metadata.Schemas[0].MaterializedViews)
+ require.ElementsMatch(t, []string{"t", "v", "ext", "mlog$_t", container}, syncedRelationNames(metadata.Schemas[0]))
+
+ // The same server registered as a MySQL instance keeps the MySQL sync result.
+ asMySQL := server.open(ctx, t, storepb.Engine_MYSQL, server.user, server.password, database)
+ metadata, err = asMySQL.SyncDBSchema(ctx)
+ require.NoError(t, err)
+ require.Empty(t, metadata.Schemas[0].ExternalTables)
+ require.Empty(t, metadata.Schemas[0].MaterializedViews)
+ require.ElementsMatch(t, []string{"t", "v", "mlog$_t", container}, syncedRelationNames(metadata.Schemas[0]))
+}
+
+// oceanBaseObjectCensus lists every object the catalog holds in the database except the
+// database itself and its indexes.
+func oceanBaseObjectCensus(ctx context.Context, t *testing.T, driver *Driver, database string) []string {
+ t.Helper()
+ rows, err := driver.db.QueryContext(ctx,
+ "SELECT OBJECT_NAME FROM oceanbase.DBA_OBJECTS WHERE OWNER = ? AND OBJECT_TYPE NOT IN ('DATABASE', 'INDEX')", database)
+ require.NoError(t, err)
+ defer rows.Close()
+ var names []string
+ for rows.Next() {
+ var name string
+ require.NoError(t, rows.Scan(&name))
+ names = append(names, name)
+ }
+ require.NoError(t, rows.Err())
+ return names
+}
+
+func syncedRelationNames(schema *metadatapb.SchemaMetadata) []string {
+ var names []string
+ for _, table := range schema.Tables {
+ names = append(names, table.Name)
+ }
+ for _, view := range schema.Views {
+ names = append(names, view.Name)
+ }
+ for _, table := range schema.ExternalTables {
+ names = append(names, table.Name)
+ }
+ for _, view := range schema.MaterializedViews {
+ names = append(names, view.Name)
+ }
+ return names
+}
+
+func requireMySQLErrorNumber(t *testing.T, err error, number uint16, msgAndArgs ...any) {
+ t.Helper()
+ var mysqlErr *mysql.MySQLError
+ require.ErrorAs(t, err, &mysqlErr, msgAndArgs...)
+ require.Equal(t, number, mysqlErr.Number, msgAndArgs...)
+}
+
+var (
+ // testOceanBaseMViewRows are the DBA_MVIEWS rows the fake OceanBase server returns.
+ testOceanBaseMViewRows [][]driver.Value
+ // testOceanBaseMViewErr, when set, fails the DBA_MVIEWS query.
+ testOceanBaseMViewErr error
+)
+
+func init() {
+ sql.Register("test_oceanbase_mviews", testOceanBaseMViewDriver{})
+}
+
+type testOceanBaseMViewDriver struct{}
+
+func (testOceanBaseMViewDriver) Open(string) (driver.Conn, error) {
+ return testOceanBaseMViewConn{}, nil
+}
+
+type testOceanBaseMViewConn struct{}
+
+func (testOceanBaseMViewConn) Prepare(string) (driver.Stmt, error) {
+ return nil, driver.ErrSkip
+}
+
+func (testOceanBaseMViewConn) Close() error {
+ return nil
+}
+
+func (testOceanBaseMViewConn) Begin() (driver.Tx, error) {
+ return nil, driver.ErrSkip
+}
+
+func (testOceanBaseMViewConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {
+ if testOceanBaseMViewErr != nil {
+ return nil, testOceanBaseMViewErr
+ }
+ return &testExplainResultRows{columns: []string{"MVIEW_NAME", "CONTAINER_NAME", "QUERY"}, rows: testOceanBaseMViewRows}, nil
+}
+
+func newTestOceanBaseMViewDriver(t *testing.T, rows [][]driver.Value, queryErr error) *Driver {
+ t.Helper()
+ testOceanBaseMViewRows = rows
+ testOceanBaseMViewErr = queryErr
+ db, err := sql.Open("test_oceanbase_mviews", "")
+ require.NoError(t, err)
+ t.Cleanup(func() { _ = db.Close() })
+ return &Driver{dbType: storepb.Engine_OCEANBASE, db: db}
+}
+
+func TestListOceanBaseMaterializedViews(t *testing.T) {
+ ctx := context.Background()
+ containerIndex := &metadatapb.IndexMetadata{Name: "idx_a", Type: "BTREE", Expressions: []string{"a"}, KeyLength: []int64{-1}, Descending: []bool{false}, Visible: true}
+ tables := []*metadatapb.TableMetadata{
+ {Name: "t"},
+ {Name: "__mv_container_500006", Comment: "materialized", Indexes: []*metadatapb.IndexMetadata{containerIndex}},
+ }
+
+ d := newTestOceanBaseMViewDriver(t, [][]driver.Value{
+ {"mv", "__mv_container_500006", "select `db`.`t`.`a` AS `a` from `db`.`t`"},
+ {"mv_unlisted_container", "__mv_container_500009", "select 1 AS `1`"},
+ }, nil)
+ views, err := d.listOceanBaseMaterializedViews(ctx, "db", tables)
+ require.NoError(t, err)
+ require.Empty(t, cmp.Diff([]*metadatapb.MaterializedViewMetadata{
+ {Name: "mv", Definition: "select `db`.`t`.`a` AS `a` from `db`.`t`", Comment: "materialized", Indexes: []*metadatapb.IndexMetadata{containerIndex}, SkipDump: true},
+ {Name: "mv_unlisted_container", Definition: "select 1 AS `1`", SkipDump: true},
+ }, views, protocmp.Transform()))
+ require.NotSame(t, containerIndex, views[0].Indexes[0])
+
+ // A failed read, such as one without SELECT on the oceanbase database, does not fail the sync.
+ d = newTestOceanBaseMViewDriver(t, nil, &mysql.MySQLError{Number: 1142, Message: "SELECT command denied to user 'reader'@'%' for table 'dba_mviews'"})
+ views, err = d.listOceanBaseMaterializedViews(ctx, "db", tables)
+ require.NoError(t, err)
+ require.Empty(t, views)
+
+ canceled, cancel := context.WithCancel(ctx)
+ cancel()
+ _, err = d.listOceanBaseMaterializedViews(canceled, "db", tables)
+ require.ErrorIs(t, err, context.Canceled)
+}