Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(oceanbase): sync external tables and materialized views (#21685)
rebelice43fbe0afdd5c
diff --git a/backend/plugin/db/mysql/mysql.go b/backend/plugin/db/mysql/mysql.goindex 0d029352b7..8e984eab00 100644--- a/backend/plugin/db/mysql/mysql.go+++ b/backend/plugin/db/mysql/mysql.go@@ -43,6 +43,8 @@ var (// MariaDB-only TABLE_TYPE values. Both kinds share the table namespace.mariaDBSystemVersionedTableType = "SYSTEM VERSIONED"mariaDBSequenceTableType = "SEQUENCE"+ // OceanBase-only TABLE_TYPE value; the kind shares the table namespace.+ oceanBaseExternalTableType = "EXTERNAL TABLE"_ db.Driver = (*Driver)(nil))diff --git a/backend/plugin/db/mysql/sync.go b/backend/plugin/db/mysql/sync.goindex 9985ed4724..fcf1cdf392 100644--- a/backend/plugin/db/mysql/sync.go+++ b/backend/plugin/db/mysql/sync.go@@ -17,6 +17,7 @@ import (metadatapb "github.com/bytebase/omni/metadata""golang.org/x/text/encoding/charmap""golang.org/x/text/transform"+ "google.golang.org/protobuf/proto""github.com/blang/semver/v4""github.com/pkg/errors"@@ -586,6 +587,7 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe// Keyed on the registered engine, not the server: an instance registered as MySQL// keeps its MySQL sync result even when the server is MariaDB.mariaDBInstance := d.dbType == storepb.Engine_MARIADB+ oceanBaseInstance := d.dbType == storepb.Engine_OCEANBASEvar sequences []*metadatapb.SequenceMetadatafor tableRows.Next() {var tableName, tableType, engine, collation, createOptions, comment, charset string@@ -669,6 +671,11 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMeDataType: mariaDBSequenceDataType(columnMap[key]),Comment: comment,})+ case oceanBaseInstance && tableType == oceanBaseExternalTableType:+ schemaMetadata.ExternalTables = append(schemaMetadata.ExternalTables, &metadatapb.ExternalTableMetadata{+ Name: tableName,+ Columns: columnMap[key],+ })default:// Unknown table type, skip}@@ -680,6 +687,13 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMereturn nil, err}schemaMetadata.Sequences = sequences+ if oceanBaseInstance {+ materializedViews, err := d.listOceanBaseMaterializedViews(ctx, d.databaseName, schemaMetadata.Tables)+ if err != nil {+ return nil, err+ }+ schemaMetadata.MaterializedViews = materializedViews+ }databaseMetadata := &metadatapb.DatabaseSchemaMetadata{Name: d.databaseName,@@ -748,6 +762,56 @@ func (d *Driver) readMariaDBSequenceAttributes(ctx context.Context, databaseNamereturn nil}+// listOceanBaseMaterializedViews reads the database's materialized views. OceanBase lists one+// in neither information_schema.TABLES nor VIEWS, only its storage table __mv_container_<id>,+// which carries the view's comment and indexes and stays synced as a table. A failed read, such+// as one without SELECT on the oceanbase database, leaves the list empty; a caller can tell,+// because a container table is then synced without its materialized view.+func (d *Driver) listOceanBaseMaterializedViews(ctx context.Context, databaseName string, tables []*metadatapb.TableMetadata) ([]*metadatapb.MaterializedViewMetadata, error) {+ query := `+ SELECT MVIEW_NAME, CONTAINER_NAME, QUERY+ FROM oceanbase.DBA_MVIEWS+ WHERE OWNER = ?+ ORDER BY MVIEW_NAME`+ rows, err := d.db.QueryContext(ctx, query, databaseName)+ if err != nil {+ if ctx.Err() != nil {+ return nil, ctx.Err()+ }+ slog.Warn("failed to list OceanBase materialized views", slog.String("database", databaseName), log.BBError(err))+ return nil, nil+ }+ defer rows.Close()+ containers := make(map[string]*metadatapb.TableMetadata, len(tables))+ for _, table := range tables {+ containers[table.Name] = table+ }+ var views []*metadatapb.MaterializedViewMetadata+ for rows.Next() {+ var name, containerName, definition string+ if err := rows.Scan(&name, &containerName, &definition); err != nil {+ return nil, err+ }+ view := &metadatapb.MaterializedViewMetadata{+ Name: name,+ Definition: definition,+ // No MySQL-family DDL writer renders OceanBase's CREATE MATERIALIZED VIEW.+ SkipDump: true,+ }+ if container, ok := containers[containerName]; ok {+ view.Comment = container.Comment+ for _, index := range container.Indexes {+ view.Indexes = append(view.Indexes, proto.CloneOf(index))+ }+ }+ views = append(views, view)+ }+ if err := rows.Err(); err != nil {+ return nil, util.FormatErrorWithQuery(err, query)+ }+ return views, nil+}+func (d *Driver) getEventList(ctx context.Context, databaseName string) ([]*metadatapb.EventMetadata, error) {listEventsQuery := `SELECTdiff --git a/backend/plugin/db/mysql/sync_oceanbase_test.go b/backend/plugin/db/mysql/sync_oceanbase_test.gonew file mode 100644index 0000000000..c0ec31fc07--- /dev/null+++ b/backend/plugin/db/mysql/sync_oceanbase_test.go@@ -0,0 +1,291 @@+package mysql++import (+ "context"+ "database/sql"+ "database/sql/driver"+ "fmt"+ "maps"+ "net"+ "os"+ "slices"+ "strings"+ "testing"+ "time"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/go-sql-driver/mysql"+ "github.com/google/go-cmp/cmp"+ "github.com/stretchr/testify/require"+ "google.golang.org/protobuf/testing/protocmp"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+)++// oceanBaseDSNEnv names a MySQL-mode OceanBase tenant user that can create databases and users,+// such as root@test:@tcp(127.0.0.1:2881)/. OceanBase needs about 8 GB, more than CI has; omni's+// scripts/oceanbase-oracle.sh boots one with secure_file_priv set and exports the variable.+const oceanBaseDSNEnv = "OCEANBASE_WALK_DSN"++type oceanBaseTestServer struct {+ host, port string+ user string+ password string+ // tenant is the login suffix after "@" that every user of the tenant signs in with.+ tenant string+}++func newOceanBaseTestServer(t *testing.T) oceanBaseTestServer {+ t.Helper()+ dsn := os.Getenv(oceanBaseDSNEnv)+ if dsn == "" {+ t.Skipf("%s is not set", oceanBaseDSNEnv)+ }+ cfg, err := mysql.ParseDSN(dsn)+ require.NoError(t, err)+ host, port, err := net.SplitHostPort(cfg.Addr)+ require.NoError(t, err)+ _, tenant, _ := strings.Cut(cfg.User, "@")+ return oceanBaseTestServer{host: host, port: port, user: cfg.User, password: cfg.Passwd, tenant: tenant}+}++func (s oceanBaseTestServer) open(ctx context.Context, t *testing.T, engine storepb.Engine, user, password, database string) *Driver {+ t.Helper()+ d, err := newDriver().Open(ctx, engine, db.ConnectionConfig{+ DataSource: &storepb.DataSource{+ Type: storepb.DataSourceType_ADMIN,+ Username: user,+ Host: s.host,+ Port: s.port,+ },+ Password: password,+ ConnectionContext: db.ConnectionContext{DatabaseName: database},+ })+ require.NoError(t, err)+ t.Cleanup(func() { require.NoError(t, d.Close(ctx)) })+ driver, ok := d.(*Driver)+ require.True(t, ok)+ return driver+}++// TestSyncOceanBaseTableNamespace covers the OceanBase objects that share the table namespace+// with base tables. A consumer can only conclude that a table name is free if sync lists every+// one of them.+func TestSyncOceanBaseTableNamespace(t *testing.T) {+ server := newOceanBaseTestServer(t)+ ctx := context.Background()+ startNano := time.Now().UnixNano()+ database := fmt.Sprintf("sync_namespace_%d", startNano)+ reader := fmt.Sprintf("sync_reader_%d", startNano)++ admin := server.open(ctx, t, storepb.Engine_OCEANBASE, server.user, server.password, "")+ _, err := admin.db.ExecContext(ctx, fmt.Sprintf("CREATE DATABASE `%s`", database))+ require.NoError(t, err)+ t.Cleanup(func() {+ _, err := admin.db.ExecContext(ctx, fmt.Sprintf("DROP DATABASE `%s`", database))+ require.NoError(t, err)+ _, err = admin.db.ExecContext(ctx, fmt.Sprintf("DROP USER IF EXISTS '%s'@'%%'", reader))+ require.NoError(t, err)+ })++ driver := server.open(ctx, t, storepb.Engine_OCEANBASE, server.user, server.password, database)+ for _, statement := range []string{+ "CREATE TABLE t (id INT PRIMARY KEY, a INT) COMMENT 'base'",+ "CREATE VIEW v AS SELECT id, a FROM t",+ "CREATE EXTERNAL TABLE ext (c1 INT, c2 VARCHAR(10)) LOCATION = '/tmp' FORMAT = (TYPE = 'CSV')",+ "CREATE MATERIALIZED VIEW mv COMMENT 'materialized' AS SELECT id, a FROM t",+ "CREATE INDEX idx_mv_a ON mv (a)",+ "CREATE MATERIALIZED VIEW LOG ON t WITH PRIMARY KEY (a) INCLUDING NEW VALUES",+ "CREATE SEQUENCE seq",+ } {+ _, err := driver.db.ExecContext(ctx, statement)+ require.NoError(t, err, statement)+ }+ var container string+ require.NoError(t, driver.db.QueryRowContext(ctx,+ "SELECT CONTAINER_NAME FROM oceanbase.DBA_MVIEWS WHERE OWNER = ? AND MVIEW_NAME = 'mv'", database,+ ).Scan(&container))++ census := oceanBaseObjectCensus(ctx, t, driver, database)+ require.ElementsMatch(t, []string{"t", "v", "ext", "mv", "mlog$_t", container}, census)+ for _, name := range census {+ _, err := driver.db.ExecContext(ctx, fmt.Sprintf("CREATE TABLE `%s` (z INT)", name))+ requireMySQLErrorNumber(t, err, 1050, name)+ }+ // A sequence has a namespace of its own.+ _, err = driver.db.ExecContext(ctx, "CREATE TABLE seq (z INT)")+ require.NoError(t, err)+ _, err = driver.db.ExecContext(ctx, "DROP TABLE seq")+ require.NoError(t, err)++ metadata, err := driver.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Len(t, metadata.Schemas, 1)+ schema := metadata.Schemas[0]+ require.ElementsMatch(t, census, syncedRelationNames(schema))++ tables := make(map[string]*metadatapb.TableMetadata)+ for _, table := range schema.Tables {+ tables[table.Name] = table+ }+ require.ElementsMatch(t, []string{"t", "mlog$_t", container}, slices.Collect(maps.Keys(tables)))+ require.Equal(t, "materialized", tables[container].Comment)++ require.Len(t, schema.ExternalTables, 1)+ require.Equal(t, "ext", schema.ExternalTables[0].Name)+ require.Equal(t, []string{"c1", "c2"}, columnNames(schema.ExternalTables[0].Columns))++ require.Len(t, schema.MaterializedViews, 1)+ mv := schema.MaterializedViews[0]+ require.Equal(t, "mv", mv.Name)+ require.Equal(t, fmt.Sprintf("select `%[1]s`.`t`.`id` AS `id`,`%[1]s`.`t`.`a` AS `a` from `%[1]s`.`t`", database), mv.Definition)+ require.Equal(t, "materialized", mv.Comment)+ require.True(t, mv.SkipDump)+ require.Len(t, mv.Indexes, 1)+ require.Equal(t, "idx_mv_a", mv.Indexes[0].Name)+ require.Equal(t, []string{"a"}, mv.Indexes[0].Expressions)+ require.Empty(t, cmp.Diff(tables[container].Indexes, mv.Indexes, protocmp.Transform()))++ // Without SELECT on the oceanbase database the materialized views are unreadable; the+ // container table, synced without its view, shows that the list is incomplete.+ _, err = admin.db.ExecContext(ctx, fmt.Sprintf("CREATE USER '%s'@'%%' IDENTIFIED BY 'Reader_pw1'", reader))+ require.NoError(t, err)+ _, err = admin.db.ExecContext(ctx, fmt.Sprintf("GRANT SELECT, SHOW VIEW ON `%s`.* TO '%s'@'%%'", database, reader))+ require.NoError(t, err)+ asReader := server.open(ctx, t, storepb.Engine_OCEANBASE, reader+"@"+server.tenant, "Reader_pw1", database)+ metadata, err = asReader.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Empty(t, metadata.Schemas[0].MaterializedViews)+ require.ElementsMatch(t, []string{"t", "v", "ext", "mlog$_t", container}, syncedRelationNames(metadata.Schemas[0]))++ // The same server registered as a MySQL instance keeps the MySQL sync result.+ asMySQL := server.open(ctx, t, storepb.Engine_MYSQL, server.user, server.password, database)+ metadata, err = asMySQL.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Empty(t, metadata.Schemas[0].ExternalTables)+ require.Empty(t, metadata.Schemas[0].MaterializedViews)+ require.ElementsMatch(t, []string{"t", "v", "mlog$_t", container}, syncedRelationNames(metadata.Schemas[0]))+}++// oceanBaseObjectCensus lists every object the catalog holds in the database except the+// database itself and its indexes.+func oceanBaseObjectCensus(ctx context.Context, t *testing.T, driver *Driver, database string) []string {+ t.Helper()+ rows, err := driver.db.QueryContext(ctx,+ "SELECT OBJECT_NAME FROM oceanbase.DBA_OBJECTS WHERE OWNER = ? AND OBJECT_TYPE NOT IN ('DATABASE', 'INDEX')", database)+ require.NoError(t, err)+ defer rows.Close()+ var names []string+ for rows.Next() {+ var name string+ require.NoError(t, rows.Scan(&name))+ names = append(names, name)+ }+ require.NoError(t, rows.Err())+ return names+}++func syncedRelationNames(schema *metadatapb.SchemaMetadata) []string {+ var names []string+ for _, table := range schema.Tables {+ names = append(names, table.Name)+ }+ for _, view := range schema.Views {+ names = append(names, view.Name)+ }+ for _, table := range schema.ExternalTables {+ names = append(names, table.Name)+ }+ for _, view := range schema.MaterializedViews {+ names = append(names, view.Name)+ }+ return names+}++func requireMySQLErrorNumber(t *testing.T, err error, number uint16, msgAndArgs ...any) {+ t.Helper()+ var mysqlErr *mysql.MySQLError+ require.ErrorAs(t, err, &mysqlErr, msgAndArgs...)+ require.Equal(t, number, mysqlErr.Number, msgAndArgs...)+}++var (+ // testOceanBaseMViewRows are the DBA_MVIEWS rows the fake OceanBase server returns.+ testOceanBaseMViewRows [][]driver.Value+ // testOceanBaseMViewErr, when set, fails the DBA_MVIEWS query.+ testOceanBaseMViewErr error+)++func init() {+ sql.Register("test_oceanbase_mviews", testOceanBaseMViewDriver{})+}++type testOceanBaseMViewDriver struct{}++func (testOceanBaseMViewDriver) Open(string) (driver.Conn, error) {+ return testOceanBaseMViewConn{}, nil+}++type testOceanBaseMViewConn struct{}++func (testOceanBaseMViewConn) Prepare(string) (driver.Stmt, error) {+ return nil, driver.ErrSkip+}++func (testOceanBaseMViewConn) Close() error {+ return nil+}++func (testOceanBaseMViewConn) Begin() (driver.Tx, error) {+ return nil, driver.ErrSkip+}++func (testOceanBaseMViewConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {+ if testOceanBaseMViewErr != nil {+ return nil, testOceanBaseMViewErr+ }+ return &testExplainResultRows{columns: []string{"MVIEW_NAME", "CONTAINER_NAME", "QUERY"}, rows: testOceanBaseMViewRows}, nil+}++func newTestOceanBaseMViewDriver(t *testing.T, rows [][]driver.Value, queryErr error) *Driver {+ t.Helper()+ testOceanBaseMViewRows = rows+ testOceanBaseMViewErr = queryErr+ db, err := sql.Open("test_oceanbase_mviews", "")+ require.NoError(t, err)+ t.Cleanup(func() { _ = db.Close() })+ return &Driver{dbType: storepb.Engine_OCEANBASE, db: db}+}++func TestListOceanBaseMaterializedViews(t *testing.T) {+ ctx := context.Background()+ containerIndex := &metadatapb.IndexMetadata{Name: "idx_a", Type: "BTREE", Expressions: []string{"a"}, KeyLength: []int64{-1}, Descending: []bool{false}, Visible: true}+ tables := []*metadatapb.TableMetadata{+ {Name: "t"},+ {Name: "__mv_container_500006", Comment: "materialized", Indexes: []*metadatapb.IndexMetadata{containerIndex}},+ }++ d := newTestOceanBaseMViewDriver(t, [][]driver.Value{+ {"mv", "__mv_container_500006", "select `db`.`t`.`a` AS `a` from `db`.`t`"},+ {"mv_unlisted_container", "__mv_container_500009", "select 1 AS `1`"},+ }, nil)+ views, err := d.listOceanBaseMaterializedViews(ctx, "db", tables)+ require.NoError(t, err)+ require.Empty(t, cmp.Diff([]*metadatapb.MaterializedViewMetadata{+ {Name: "mv", Definition: "select `db`.`t`.`a` AS `a` from `db`.`t`", Comment: "materialized", Indexes: []*metadatapb.IndexMetadata{containerIndex}, SkipDump: true},+ {Name: "mv_unlisted_container", Definition: "select 1 AS `1`", SkipDump: true},+ }, views, protocmp.Transform()))+ require.NotSame(t, containerIndex, views[0].Indexes[0])++ // A failed read, such as one without SELECT on the oceanbase database, does not fail the sync.+ d = newTestOceanBaseMViewDriver(t, nil, &mysql.MySQLError{Number: 1142, Message: "SELECT command denied to user 'reader'@'%' for table 'dba_mviews'"})+ views, err = d.listOceanBaseMaterializedViews(ctx, "db", tables)+ require.NoError(t, err)+ require.Empty(t, views)++ canceled, cancel := context.WithCancel(ctx)+ cancel()+ _, err = d.listOceanBaseMaterializedViews(canceled, "db", tables)+ require.ErrorIs(t, err, context.Canceled)+}