Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
feat(starrocks): sync materialized views via information_schema.materialized_views (#20619)
Vincent Huang5d8789c7e211
diff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.goindex 8f0bf95192..ad5270e10c 100644--- a/backend/common/testcontainer/testcontainer.go+++ b/backend/common/testcontainer/testcontainer.go@@ -5,6 +5,8 @@ import ("database/sql""fmt""log/slog"+ "runtime"+ "strings""testing""time"@@ -253,6 +255,132 @@ func GetTestOracleContainer(ctx context.Context, t testing.TB) *Container {return container}+// GetStarRocksContainer creates a StarRocks (allin1) container for testing. The allin1+// image bundles the FE and BE; information_schema is served by the BE, so this waits until+// the backend actually serves queries rather than only the FE port being open.+//+// NOTE: requires an amd64 host — the StarRocks BE has no working arm64 build (it fails to+// come alive under emulation). bytebase CI runs on amd64; locally, run without -short on an+// amd64 machine, or the test is skipped via testing.Short().+func GetStarRocksContainer(ctx context.Context) (retC *Container, retErr error) {+ req := testcontainers.ContainerRequest{+ Image: "starrocks/allin1-ubuntu:3.4.10",+ ExposedPorts: []string{"9030/tcp"},+ WaitingFor: wait.ForListeningPort("9030/tcp").WithStartupTimeout(5 * time.Minute),+ }+ c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{+ ContainerRequest: req,+ Started: true,+ })+ if err != nil {+ return nil, err+ }+ host, err := c.Host(ctx)+ if err != nil {+ return nil, err+ }+ port, err := c.MappedPort(ctx, "9030/tcp")+ if err != nil {+ return nil, err+ }+ dsn := fmt.Sprintf("root@tcp(%s:%s)/?multiStatements=true", host, port.Port())+ db, err := sql.Open("mysql", dsn)+ if err != nil {+ return nil, err+ }+ defer func() {+ if retErr != nil {+ db.Close()+ }+ }()+ if err := waitStarRocksReady(ctx, db); err != nil {+ return nil, err+ }+ return &Container{+ container: c,+ host: host,+ port: port.Port(),+ db: db,+ }, nil+}++// GetTestStarRocksContainer creates a StarRocks container and fails the test on error. It+// skips on non-amd64 hosts: the StarRocks all-in-one BE has no working arm64 build (it never+// comes alive under emulation), so the readiness wait would otherwise time out.+func GetTestStarRocksContainer(ctx context.Context, t testing.TB) *Container {+ t.Helper()+ if runtime.GOARCH != "amd64" {+ t.Skipf("StarRocks requires an amd64 host; the all-in-one BE has no working arm64 build (GOARCH=%s)", runtime.GOARCH)+ }+ container, err := GetStarRocksContainer(ctx)+ if err != nil {+ t.Fatalf("failed to create StarRocks container: %v", err)+ }+ return container+}++// waitStarRocksReady blocks until at least one StarRocks backend is registered and alive,+// which is required before tablet-allocating DDL (CREATE TABLE) succeeds. The check is+// read-only on purpose: issuing DDL during allin1's first-boot disrupts backend+// registration, so poll SHOW BACKENDS rather than probing with CREATE TABLE.+func waitStarRocksReady(ctx context.Context, db *sql.DB) error {+ ticker := time.NewTicker(3 * time.Second)+ defer ticker.Stop()+ timeout := time.After(10 * time.Minute)+ for {+ select {+ case <-ticker.C:+ if starRocksBackendAlive(ctx, db) {+ return nil+ }+ case <-timeout:+ return errors.Errorf("StarRocks backend did not become ready")+ }+ }+}++// starRocksBackendAlive reports whether SHOW BACKENDS lists a backend with Alive=true.+func starRocksBackendAlive(ctx context.Context, db *sql.DB) bool {+ rows, err := db.QueryContext(ctx, "SHOW BACKENDS")+ if err != nil {+ return false+ }+ defer rows.Close()+ cols, err := rows.Columns()+ if err != nil {+ return false+ }+ aliveIdx := -1+ for i, c := range cols {+ if strings.EqualFold(c, "Alive") {+ aliveIdx = i+ break+ }+ }+ if aliveIdx < 0 {+ return false+ }+ alive := false+ for rows.Next() {+ vals := make([]sql.RawBytes, len(cols))+ ptrs := make([]any, len(cols))+ for i := range vals {+ ptrs[i] = &vals[i]+ }+ if err := rows.Scan(ptrs...); err != nil {+ return false+ }+ if strings.EqualFold(string(vals[aliveIdx]), "true") {+ alive = true+ break+ }+ }+ if err := rows.Err(); err != nil {+ return false+ }+ return alive+}+// GetMSSQLContainer creates a Microsoft SQL Server container for testingfunc GetMSSQLContainer(ctx context.Context) (retC *Container, retErr error) {req := testcontainers.ContainerRequest{diff --git a/backend/plugin/db/starrocks/dump.go b/backend/plugin/db/starrocks/dump.goindex 2365d841b3..5b83480766 100644--- a/backend/plugin/db/starrocks/dump.go+++ b/backend/plugin/db/starrocks/dump.go@@ -98,8 +98,13 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *storepb.DatabaseSchem// Disable foreign key check.// mysqldump uses the same mechanism. When there is any schema or data dependency, we have to disable// the unique and foreign key check so that the restoring will not fail.- if _, err := io.WriteString(out, disableUniqueAndForeignKeyCheckStmt); err != nil {- return err+ // StarRocks has no MySQL UNIQUE_CHECKS/FOREIGN_KEY_CHECKS session variables (and does not+ // enforce foreign keys), so emitting the guard makes the dump fail to replay with+ // "Unknown system variable 'UNIQUE_CHECKS'".+ if d.dbType != storepb.Engine_STARROCKS {+ if _, err := io.WriteString(out, disableUniqueAndForeignKeyCheckStmt); err != nil {+ return err+ }}// Table and view statement.@@ -150,18 +155,15 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *storepb.DatabaseSchemreturn err}}- // Construct final views and materialized views.- for _, tbl := range tables {- if tbl.TableType != viewTableType && tbl.TableType != materializedViewType {- continue- }- // The temporary view just created above were used to satisfy the schema dependency. See comment above.- // We have to drop the temporary and incorrect view here to recreate the final and correct one.- dropStmt := "DROP VIEW IF EXISTS `%s`;\n"- if tbl.TableType == materializedViewType {- dropStmt = "DROP MATERIALIZED VIEW IF EXISTS `%s`;\n"- }- if _, err := io.WriteString(out, fmt.Sprintf(dropStmt, tbl.Name)); err != nil {+ // Construct final views, then materialized views (see finalEmitOrder: an MV is+ // materialized from its sources at CREATE time, so its source views must be real+ // first, not the temporary SELECT 1 placeholders).+ for _, tbl := range finalEmitOrder(tables) {+ // The temporary placeholder created above is always a regular view (even for+ // materialized views — see getTemporaryMaterializedView), so it must be dropped+ // with DROP VIEW. DROP MATERIALIZED VIEW would not match the placeholder, leaving+ // it in the shared namespace so the real CREATE MATERIALIZED VIEW below collides.+ if _, err := io.WriteString(out, dropPlaceholderStmt(tbl.Name)); err != nil {return err}if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.Statement)); err != nil {@@ -191,14 +193,49 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *storepb.DatabaseSchem}}- // Restore foreign key check.- if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {- return err+ // Restore foreign key check (skipped for StarRocks, which lacks these variables).+ if d.dbType != storepb.Engine_STARROCKS {+ if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {+ return err+ }}return nil}+// finalEmitOrder returns the views and materialized views in the order their real+// definitions must be emitted: all regular views first, then materialized views, each+// group keeping its original relative order. information_schema.tables is unordered, and+// a materialized view can be defined on a regular view and is materialized from its+// sources at CREATE time, so its source views must already be real (not the temporary+// SELECT 1 placeholders). Base tables are emitted separately and are excluded here.+//+// MVs keep their incoming order relative to each other, so a materialized view built on+// another materialized view (nested async MVs) is not yet dependency-ordered and could+// restore inactive if listed before its parent — tracked in BYT-9718.+func finalEmitOrder(tables []*TableSchema) []*TableSchema {+ var views, materializedViews []*TableSchema+ for _, tbl := range tables {+ switch tbl.TableType {+ case viewTableType:+ views = append(views, tbl)+ case materializedViewType:+ materializedViews = append(materializedViews, tbl)+ default:+ // Base tables and any other types are emitted elsewhere; skip here.+ }+ }+ return append(views, materializedViews...)+}++// dropPlaceholderStmt returns the statement that drops the temporary placeholder emitted+// for a view or materialized view before its real definition. The placeholder is always a+// regular view (see getTemporaryMaterializedView), so it is dropped with DROP VIEW+// regardless of the final object type.+func dropPlaceholderStmt(name string) string {+ return fmt.Sprintf("DROP VIEW IF EXISTS `%s`;\n", name)+}+func getTemporaryView(name string, columns []string) string {var parts []stringfor _, col := range columns {@@ -276,35 +313,13 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchemareturn nil, err}- // For Doris, get materialized views using mv_infos() and mark them appropriately- materializedViewNames := make(map[string]bool)- if dbType == storepb.Engine_DORIS {- mvQuery := fmt.Sprintf(`SELECT Name FROM mv_infos("database"="%s")`, dbName)- mvRows, err := db.Query(mvQuery)- if err != nil {- // mv_infos() might not be available in older versions, just log and continue- slog.Debug("failed to query mv_infos(), might not be supported", slog.String("error", err.Error()))- } else {- defer mvRows.Close()- for mvRows.Next() {- var mvName string- if err := mvRows.Scan(&mvName); err != nil {- return nil, err- }- materializedViewNames[mvName] = true- }- if err := mvRows.Err(); err != nil {- return nil, err- }- }- }-- // Update table types for materialized views that are marked as BASE TABLE- for _, tbl := range tables {- if tbl.TableType == baseTableType && materializedViewNames[tbl.Name] {- tbl.TableType = materializedViewType- }+ // Materialized views need a separate catalog lookup and re-tagging so the dump emits+ // SHOW CREATE MATERIALIZED VIEW for them (Doris reports them as BASE TABLE, StarRocks as VIEW).+ materializedViewNames, err := getMaterializedViewNames(db, dbName, dbType)+ if err != nil {+ return nil, err}+ markMaterializedViews(tables, materializedViewNames)var result []*TableSchemafor _, tbl := range tables {@@ -334,6 +349,69 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchemareturn result, nil}+// getMaterializedViewNames returns the set of materialized-view names in the database.+// Doris and StarRocks expose them through different catalogs; the query is best-effort+// (an unsupported engine/version logs and yields an empty set). StarRocks sync rollups+// (REFRESH_TYPE='ROLLUP') are excluded — they are rollup indexes on the base table, not+// standalone views.+func getMaterializedViewNames(db *sql.DB, dbName string, dbType storepb.Engine) (map[string]bool, error) {+ names := make(map[string]bool)++ var query string+ switch dbType {+ case storepb.Engine_DORIS:+ query = fmt.Sprintf(`SELECT Name FROM mv_infos("database"="%s")`, dbName)+ case storepb.Engine_STARROCKS:+ // IFNULL guards the bare-string scan below: database/sql errors on a NULL->string scan.+ query = fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, dbName)+ default:+ return names, nil+ }++ rows, err := db.Query(query)+ if err != nil {+ // The catalog may be unavailable on older engine versions; log and continue.+ slog.Debug("failed to query materialized views, might not be supported", slog.String("error", err.Error()))+ return names, nil+ }+ defer rows.Close()++ for rows.Next() {+ var name string+ if dbType == storepb.Engine_STARROCKS {+ var refreshType string+ if err := rows.Scan(&name, &refreshType); err != nil {+ return nil, err+ }+ if isSyncRollup(refreshType) {+ continue+ }+ } else if err := rows.Scan(&name); err != nil {+ return nil, err+ }+ names[name] = true+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return names, nil+}++// markMaterializedViews re-tags rows that are actually materialized views so the dump+// emits SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE' and+// StarRocks as 'VIEW'; either becomes 'MATERIALIZED VIEW' when its name is in the+// materialized-view set. Regular tables and views are left untouched.+func markMaterializedViews(tables []*TableSchema, materializedViewNames map[string]bool) {+ for _, tbl := range tables {+ if !materializedViewNames[tbl.Name] {+ continue+ }+ if tbl.TableType == baseTableType || tbl.TableType == viewTableType {+ tbl.TableType = materializedViewType+ }+ }+}+// getTableStmt gets the create statement of a table.func getTableStmt(db *sql.DB, dbName, tblName, tblType string) (string, error) {switch tblType {diff --git a/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go b/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.gonew file mode 100644index 0000000000..1cf3091538--- /dev/null+++ b/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go@@ -0,0 +1,89 @@+package starrocks++import (+ "bytes"+ "context"+ "database/sql"+ "fmt"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// TestDump_StarRocksMaterializedViewRoundTrip locks the two dump fixes (BYT-9689) against a+// real StarRocks (amd64-only; skipped in -short). A materialized view defined on a regular+// view is dumped, then replayed into a fresh database of the same name, and must come back:+// - without a name collision (P1: the temporary placeholder is dropped as a view, so the+// real CREATE MATERIALIZED VIEW does not collide with a leftover placeholder), and+// - IS_ACTIVE=true (P2: the MV is emitted after its source view, so it is created against+// the real view, not inactivated when a placeholder is dropped under it).+//+// The dump is schema-only (--no-data), so IS_ACTIVE — not row data — is the P2 signal.+func TestDump_StarRocksMaterializedViewRoundTrip(t *testing.T) {+ if testing.Short() {+ t.Skip("skipping StarRocks testcontainer test in short mode")+ }+ ctx := context.Background()+ c := testcontainer.GetTestStarRocksContainer(ctx, t)+ defer c.Close(ctx)+ db := c.GetDB()++ for _, stmt := range []string{+ "CREATE DATABASE rt",+ "CREATE TABLE rt.t (id INT, amt INT) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')",+ "INSERT INTO rt.t VALUES (1,10),(2,20)",+ "CREATE VIEW rt.vw AS SELECT id, amt FROM rt.t",+ "CREATE MATERIALIZED VIEW rt.mv REFRESH ASYNC AS SELECT id, sum(amt) AS total FROM rt.vw GROUP BY id",+ } {+ if _, err := db.ExecContext(ctx, stmt); err != nil {+ t.Fatalf("setup failed: %s: %v", stmt, err)+ }+ }+ requireMVActive(ctx, t, db, "rt", "mv")++ // Produce the dump through the real driver.+ d := &Driver{dbType: storepb.Engine_STARROCKS, db: db, databaseName: "rt"}+ var buf bytes.Buffer+ require.NoError(t, d.Dump(ctx, &buf, nil))+ dump := buf.String()+ require.Contains(t, dump, "CREATE MATERIALIZED VIEW", "dump should emit the MV")++ // Restore into a fresh same-name database. StarRocks qualifies view/MV references with+ // the original db, so the restore target must reuse the name. DROP+CREATE+USE+replay run+ // in one multi-statement Exec so USE applies to the unqualified statements on the same+ // pooled connection.+ restore := "DROP DATABASE IF EXISTS rt;\nCREATE DATABASE rt;\nUSE rt;\n" + dump+ if _, err := db.ExecContext(ctx, restore); err != nil {+ t.Fatalf("dump replay failed (P1 collision or unreplayable DDL): %v", err)+ }++ // P2: the restored MV must be active, not inactivated by the placeholder dance.+ requireMVActive(ctx, t, db, "rt", "mv")++ // And it re-syncs cleanly: mv as a materialized view, no stale placeholder.+ meta, err := d.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Len(t, meta.Schemas, 1)+ require.Len(t, meta.Schemas[0].MaterializedViews, 1)+ require.Equal(t, "mv", meta.Schemas[0].MaterializedViews[0].Name)+}++// requireMVActive waits until the named materialized view reports IS_ACTIVE=true.+func requireMVActive(ctx context.Context, t *testing.T, db *sql.DB, schema, name string) {+ t.Helper()+ q := fmt.Sprintf("SELECT IS_ACTIVE FROM information_schema.materialized_views WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s'", schema, name)+ require.Eventually(t, func() bool {+ var active string+ if err := db.QueryRowContext(ctx, q).Scan(&active); err != nil {+ return false+ }+ return active == "true"+ // information_schema.materialized_views (listMaterializedViewStatus, an FE->BE RPC)+ // is unavailable for ~tens of seconds after a fresh allin1 boot ("BE host unknown")+ // before it settles, so allow a generous window.+ }, 180*time.Second, 3*time.Second, "MV %s.%s did not become active", schema, name)+}diff --git a/backend/plugin/db/starrocks/dump_materialized_view_test.go b/backend/plugin/db/starrocks/dump_materialized_view_test.gonew file mode 100644index 0000000000..34a31ed117--- /dev/null+++ b/backend/plugin/db/starrocks/dump_materialized_view_test.go@@ -0,0 +1,60 @@+package starrocks++import (+ "testing"++ "github.com/stretchr/testify/require"+)++// BYT-9689: the dump path re-tags materialized-view rows so it emits+// SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE',+// StarRocks as 'VIEW'; both must become 'MATERIALIZED VIEW' when the name is a known+// MV, while regular tables and views are left untouched.+func TestMarkMaterializedViews(t *testing.T) {+ tables := []*TableSchema{+ {Name: "mv_doris", TableType: baseTableType}, // Doris MV reported as BASE TABLE+ {Name: "mv_starrocks", TableType: viewTableType}, // StarRocks MV reported as VIEW+ {Name: "v_regular", TableType: viewTableType}, // regular view (not an MV)+ {Name: "t_plain", TableType: baseTableType}, // plain table+ }+ markMaterializedViews(tables, map[string]bool{"mv_doris": true, "mv_starrocks": true})++ require.Equal(t, materializedViewType, tables[0].TableType)+ require.Equal(t, materializedViewType, tables[1].TableType)+ require.Equal(t, viewTableType, tables[2].TableType, "regular view must stay a view")+ require.Equal(t, baseTableType, tables[3].TableType, "plain table must stay a table")+}++// BYT-9689: the dump emits a temporary regular-view placeholder for every view AND+// materialized view (getTemporaryMaterializedView creates a CREATE VIEW), then drops it+// before emitting the real definition. The drop must therefore be DROP VIEW even for an+// MV — DROP MATERIALIZED VIEW would not match the placeholder, leaving it in the shared+// namespace so the real CREATE MATERIALIZED VIEW collides with it on replay.+func TestMaterializedViewPlaceholderIsDroppedAsView(t *testing.T) {+ placeholder := getTemporaryMaterializedView("mv1", []string{"a"})+ require.Contains(t, placeholder, "CREATE VIEW `mv1`")+ require.NotContains(t, placeholder, "CREATE MATERIALIZED VIEW")++ require.Equal(t, "DROP VIEW IF EXISTS `mv1`;\n", dropPlaceholderStmt("mv1"))+}++// BYT-9689: information_schema.tables is unordered, so an MV can be listed before a view+// it is defined on. A materialized view is materialized from its sources at CREATE time+// (unlike a lazy regular view), so all regular views must be emitted — and on restore,+// created real — before any materialized view; otherwise the MV would materialize from+// the temporary SELECT 1 placeholder. finalEmitOrder partitions views ahead of MVs while+// preserving each group's relative order, and drops base tables (emitted elsewhere).+func TestFinalEmitOrder_ViewsBeforeMaterializedViews(t *testing.T) {+ tables := []*TableSchema{+ {Name: "mv1", TableType: materializedViewType},+ {Name: "v1", TableType: viewTableType},+ {Name: "t1", TableType: baseTableType},+ {Name: "mv2", TableType: materializedViewType},+ {Name: "v2", TableType: viewTableType},+ }+ var names []string+ for _, tbl := range finalEmitOrder(tables) {+ names = append(names, tbl.Name)+ }+ require.Equal(t, []string{"v1", "v2", "mv1", "mv2"}, names)+}diff --git a/backend/plugin/db/starrocks/sync.go b/backend/plugin/db/starrocks/sync.goindex bcc15dcf5a..f0b8db4c98 100644--- a/backend/plugin/db/starrocks/sync.go+++ b/backend/plugin/db/starrocks/sync.go@@ -114,6 +114,85 @@ func (d *Driver) getServerVariable(ctx context.Context, varName string) (string,return value, nil}+// isSyncRollup reports whether a StarRocks information_schema.materialized_views row+// describes a synchronous rollup rather than a standalone (async) materialized view.+// Sync rollups are rollup indexes on the base table: they appear in that catalog with+// REFRESH_TYPE='ROLLUP' but have no information_schema.tables row, so they are excluded+// from the materialized-view set. Only the confirmed 'ROLLUP' value is excluded — async+// refresh types and any unknown/empty value are kept.+func isSyncRollup(refreshType string) bool {+ return refreshType == "ROLLUP"+}++// isMaterializedView reports whether a table row is a materialized view, based on its+// table type and membership in the materialized-view set. StarRocks reports+// materialized views with TABLE_TYPE='VIEW', Doris with 'BASE TABLE', and some setups+// with 'MATERIALIZED VIEW'; in every case, presence in materializedViewMap is the+// authoritative signal.+func isMaterializedView(tableType string, key db.TableKey, materializedViewMap map[db.TableKey]*storepb.MaterializedViewMetadata) bool {+ if _, ok := materializedViewMap[key]; !ok {+ return false+ }+ switch tableType {+ case baseTableType, viewTableType, materializedViewType:+ return true+ default:+ return false+ }+}++// getMaterializedViews returns the materialized views of the current database keyed by+// table name. Doris and StarRocks expose them through different catalogs, so the query+// and projection are engine-specific. The query is best-effort: an engine/version that+// does not support it is logged and yields an empty set rather than failing the sync.+func (d *Driver) getMaterializedViews(ctx context.Context) (map[db.TableKey]*storepb.MaterializedViewMetadata, error) {+ materializedViewMap := make(map[db.TableKey]*storepb.MaterializedViewMetadata)++ var query string+ switch d.dbType {+ case storepb.Engine_DORIS:+ // Doris exposes materialized views via the mv_infos() table-valued function.+ // https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/mv_infos+ query = fmt.Sprintf(`SELECT Name, QuerySql FROM mv_infos("database"="%s")`, d.databaseName)+ case storepb.Engine_STARROCKS:+ // StarRocks exposes materialized views via information_schema.materialized_views.+ // REFRESH_TYPE separates async MVs from synchronous rollups, which are excluded.+ // IFNULL guards the bare-string scans below: database/sql errors on a NULL->string scan.+ query = fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, ''), IFNULL(MATERIALIZED_VIEW_DEFINITION, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, d.databaseName)+ default:+ return materializedViewMap, nil+ }++ rows, err := d.db.QueryContext(ctx, query)+ if err != nil {+ // The catalog may be unavailable on older engine versions; log and continue.+ slog.Debug("failed to query materialized views, might not be supported in this version", log.BBError(err))+ return materializedViewMap, nil+ }+ defer rows.Close()++ for rows.Next() {+ materializedView := &storepb.MaterializedViewMetadata{}+ if d.dbType == storepb.Engine_STARROCKS {+ var refreshType string+ if err := rows.Scan(&materializedView.Name, &refreshType, &materializedView.Definition); err != nil {+ return nil, err+ }+ if isSyncRollup(refreshType) {+ continue+ }+ } else if err := rows.Scan(&materializedView.Name, &materializedView.Definition); err != nil {+ return nil, err+ }+ key := db.TableKey{Schema: "", Table: materializedView.Name}+ materializedViewMap[key] = materializedView+ }+ if err := rows.Err(); err != nil {+ return nil, util.FormatErrorWithQuery(err, query)+ }+ return materializedViewMap, nil+}+// SyncDBSchema syncs a single database schema.func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetadata, error) {schemaMetadata := &storepb.SchemaMetadata{@@ -226,39 +305,10 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetadreturn nil, util.FormatErrorWithQuery(err, viewQuery)}- // Query materialized view info.- // For Doris, we use the mv_infos() table-valued function to get materialized view information.- // Sync materialized views are part of the base table and don't need special handling.- materializedViewMap := make(map[db.TableKey]*storepb.MaterializedViewMetadata)-- // Only query materialized views for Doris- if d.dbType == storepb.Engine_DORIS {- // Use mv_infos() table-valued function to get materialized view information- // https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/mv_infos- materializedViewQuery := fmt.Sprintf(`SELECT Name, QuerySql FROM mv_infos("database"="%s")`, d.databaseName)- materializedViewRows, err := d.db.QueryContext(ctx, materializedViewQuery)- if err != nil {- // mv_infos() might not be available in older Doris versions, log and continue- slog.Debug("failed to query materialized views using mv_infos(), might not be supported in this version", log.BBError(err))- } else {- defer materializedViewRows.Close()- for materializedViewRows.Next() {- materializedView := &storepb.MaterializedViewMetadata{}- if err := materializedViewRows.Scan(- &materializedView.Name,- &materializedView.Definition,- ); err != nil {- return nil, err- }- key := db.TableKey{Schema: "", Table: materializedView.Name}- materializedViewMap[key] = materializedView- slog.Debug("found materialized view", slog.String("name", materializedView.Name), slog.String("database", d.databaseName))- }- if err := materializedViewRows.Err(); err != nil {- return nil, util.FormatErrorWithQuery(err, materializedViewQuery)- }- slog.Debug("total materialized views found", slog.Int("count", len(materializedViewMap)), slog.String("database", d.databaseName))- }+ // Query materialized view info (engine-specific catalog; best-effort).+ materializedViewMap, err := d.getMaterializedViews(ctx)+ if err != nil {+ return nil, err}// Query table info.@@ -303,48 +353,42 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetad}key := db.TableKey{Schema: "", Table: tableName}+ // StarRocks reports materialized views as TABLE_TYPE='VIEW', Doris as 'BASE TABLE';+ // route any row whose name is in the materialized-view set to MaterializedViews.+ if isMaterializedView(tableType, key, materializedViewMap) {+ materializedView := materializedViewMap[key]+ materializedView.Comment = comment+ schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, materializedView)+ continue+ }switch tableType {case baseTableType:- // Check if this is actually a materialized view (for Doris)- // In Doris, materialized views appear as BASE TABLE in information_schema.TABLES- if materializedView, ok := materializedViewMap[key]; ok {- slog.Debug("identified BASE TABLE as materialized view", slog.String("name", tableName), slog.String("database", d.databaseName))- materializedView.Comment = comment- schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, materializedView)- } else {- slog.Debug("adding BASE TABLE as regular table", slog.String("name", tableName), slog.String("database", d.databaseName))- tableMetadata := &storepb.TableMetadata{- Name: tableName,- Columns: columnMap[key],- Engine: engine,- Collation: collation,- RowCount: rowCount,- DataSize: dataSize,- IndexSize: indexSize,- DataFree: dataFree,- CreateOptions: createOptions,- Comment: comment,- }- if tableCollation.Valid {- tableMetadata.Collation = tableCollation.String- }- // TODO(d): add index information whenever it is available.- schemaMetadata.Tables = append(schemaMetadata.Tables, tableMetadata)+ tableMetadata := &storepb.TableMetadata{+ Name: tableName,+ Columns: columnMap[key],+ Engine: engine,+ Collation: collation,+ RowCount: rowCount,+ DataSize: dataSize,+ IndexSize: indexSize,+ DataFree: dataFree,+ CreateOptions: createOptions,+ Comment: comment,}+ if tableCollation.Valid {+ tableMetadata.Collation = tableCollation.String+ }+ // TODO(d): add index information whenever it is available.+ schemaMetadata.Tables = append(schemaMetadata.Tables, tableMetadata)case viewTableType:if view, ok := viewMap[key]; ok {view.Comment = commentschemaMetadata.Views = append(schemaMetadata.Views, view)}- case materializedViewType:- // For databases that properly set TABLE_TYPE = 'MATERIALIZED VIEW'- if materializedView, ok := materializedViewMap[key]; ok {- materializedView.Comment = comment- schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, materializedView)- }default:- // Skip unknown table types (e.g., SYSTEM VIEW, TEMPORARY, etc.)- slog.Debug("skipping unknown table type", slog.String("tableName", tableName), slog.String("tableType", tableType))+ // Includes 'MATERIALIZED VIEW' rows absent from the materialized-view set+ // and any other unrecognized type.+ slog.Debug("skipping unhandled table type", slog.String("tableName", tableName), slog.String("tableType", tableType))}}if err := tableRows.Err(); err != nil {diff --git a/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go b/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.gonew file mode 100644index 0000000000..ae004487d9--- /dev/null+++ b/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go@@ -0,0 +1,75 @@+package starrocks++import (+ "context"+ "testing"+ "time"++ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/common/testcontainer"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// TestSyncDBSchema_StarRocksMaterializedView_E2E brings up a real StarRocks (amd64-only;+// runs in CI, skipped in -short) and verifies BYT-9689 end to end: an async materialized+// view is synced under MaterializedViews with its definition, while a synchronous rollup+// (REFRESH_TYPE='ROLLUP', which has no information_schema.tables row) is excluded.+func TestSyncDBSchema_StarRocksMaterializedView_E2E(t *testing.T) {+ if testing.Short() {+ t.Skip("skipping StarRocks testcontainer test in short mode")+ }+ ctx := context.Background()+ c := testcontainer.GetTestStarRocksContainer(ctx, t)+ defer c.Close(ctx)+ db := c.GetDB()++ for _, stmt := range []string{+ "CREATE DATABASE mvtest",+ "CREATE TABLE mvtest.sales (id BIGINT, amt INT) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')",+ "INSERT INTO mvtest.sales VALUES (1,10),(2,20)",+ "CREATE MATERIALIZED VIEW mvtest.mv_async REFRESH ASYNC AS SELECT id, sum(amt) AS total FROM mvtest.sales GROUP BY id",+ "CREATE MATERIALIZED VIEW mvtest.mv_rollup AS SELECT id, sum(amt) FROM mvtest.sales GROUP BY id",+ } {+ if _, err := db.ExecContext(ctx, stmt); err != nil {+ t.Fatalf("setup failed: %s: %v", stmt, err)+ }+ }++ // The sync rollup builds asynchronously; wait until it registers in+ // information_schema.materialized_views so the test deterministically exercises the+ // rollup-exclusion path (it appears there as REFRESH_TYPE='ROLLUP' but never as a+ // information_schema.tables row).+ require.Eventually(t, func() bool {+ var n int+ if err := db.QueryRowContext(ctx,+ "SELECT COUNT(*) FROM information_schema.materialized_views WHERE TABLE_SCHEMA='mvtest' AND REFRESH_TYPE='ROLLUP'").Scan(&n); err != nil {+ return false+ }+ return n == 1+ // information_schema.materialized_views (an FE->BE RPC) is unavailable for tens of+ // seconds after a fresh allin1 boot before it settles; allow a generous window.+ }, 150*time.Second, 3*time.Second, "sync rollup did not register in materialized_views")++ d := &Driver{dbType: storepb.Engine_STARROCKS, db: db, databaseName: "mvtest"}+ meta, err := d.SyncDBSchema(ctx)+ require.NoError(t, err)+ require.Len(t, meta.Schemas, 1)+ schema := meta.Schemas[0]++ // The async MV is synced under MaterializedViews with its definition.+ require.Len(t, schema.MaterializedViews, 1, "exactly the async MV must be synced")+ require.Equal(t, "mv_async", schema.MaterializedViews[0].Name)+ require.Contains(t, schema.MaterializedViews[0].Definition, "CREATE MATERIALIZED VIEW")++ // The base table is synced as a table; the sync rollup leaks nowhere; the MV is not+ // misfiled as a regular view.+ require.Len(t, schema.Tables, 1)+ require.Equal(t, "sales", schema.Tables[0].Name)+ for _, mv := range schema.MaterializedViews {+ require.NotEqual(t, "mv_rollup", mv.Name, "sync rollup must be excluded from MaterializedViews")+ }+ for _, v := range schema.Views {+ require.NotEqual(t, "mv_async", v.Name, "MV must not be misfiled as a regular view")+ }+}diff --git a/backend/plugin/db/starrocks/sync_materialized_view_test.go b/backend/plugin/db/starrocks/sync_materialized_view_test.gonew file mode 100644index 0000000000..c8fa683e9e--- /dev/null+++ b/backend/plugin/db/starrocks/sync_materialized_view_test.go@@ -0,0 +1,45 @@+package starrocks++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+)++// BYT-9689: StarRocks reports materialized views in information_schema.tables with+// TABLE_TYPE='VIEW' and omits them from information_schema.views (verified on a live+// 3.4.10 BE-alive smoke test), while Doris reports them as 'BASE TABLE'. The shared+// driver previously only matched Doris's 'BASE TABLE' rows, so StarRocks MVs were+// dropped. Membership in the materialized-view set is the authoritative signal.+func TestIsMaterializedView(t *testing.T) {+ mvMap := map[db.TableKey]*storepb.MaterializedViewMetadata{+ {Schema: "", Table: "mv_async"}: {Name: "mv_async"},+ }+ mvKey := db.TableKey{Schema: "", Table: "mv_async"}++ // StarRocks reports the MV as VIEW; Doris as BASE TABLE; both map to an MV.+ require.True(t, isMaterializedView(viewTableType, mvKey, mvMap))+ require.True(t, isMaterializedView(baseTableType, mvKey, mvMap))+ require.True(t, isMaterializedView(materializedViewType, mvKey, mvMap))++ // A name absent from the MV set is never a materialized view.+ require.False(t, isMaterializedView(viewTableType, db.TableKey{Table: "v_regular"}, mvMap))+ require.False(t, isMaterializedView(baseTableType, db.TableKey{Table: "sales"}, mvMap))+}++// BYT-9689: StarRocks synchronous rollups are listed in+// information_schema.materialized_views with REFRESH_TYPE='ROLLUP' but, unlike async+// MVs, have no information_schema.tables row (they are rollup indexes on the base+// table). Only confirmed rollups are excluded; async MVs (ASYNC/MANUAL/...) are kept, so+// a future refresh type isn't silently dropped. The query coalesces SQL NULL to the+// empty string via IFNULL (database/sql cannot scan NULL into a string), so a NULL+// refresh type reaches this check as the empty string, which must also be kept.+func TestIsSyncRollup(t *testing.T) {+ require.True(t, isSyncRollup("ROLLUP"))+ require.False(t, isSyncRollup("ASYNC"))+ require.False(t, isSyncRollup("MANUAL"))+ require.False(t, isSyncRollup("")) // IFNULL-coalesced NULL or a genuinely empty type.+}