Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

feat(starrocks): sync materialized views via information_schema.materialized_views (#20619)

Vincent Huang5d8789c7e211
diff --git a/backend/common/testcontainer/testcontainer.go b/backend/common/testcontainer/testcontainer.go
index 8f0bf95192..ad5270e10c 100644
--- a/backend/common/testcontainer/testcontainer.go
+++ b/backend/common/testcontainer/testcontainer.go
@@ -5,6 +5,8 @@ import (
"database/sql"
"fmt"
"log/slog"
+ "runtime"
+ "strings"
"testing"
"time"
@@ -253,6 +255,132 @@ func GetTestOracleContainer(ctx context.Context, t testing.TB) *Container {
return container
}
+// GetStarRocksContainer creates a StarRocks (allin1) container for testing. The allin1
+// image bundles the FE and BE; information_schema is served by the BE, so this waits until
+// the backend actually serves queries rather than only the FE port being open.
+//
+// NOTE: requires an amd64 host — the StarRocks BE has no working arm64 build (it fails to
+// come alive under emulation). bytebase CI runs on amd64; locally, run without -short on an
+// amd64 machine, or the test is skipped via testing.Short().
+func GetStarRocksContainer(ctx context.Context) (retC *Container, retErr error) {
+ req := testcontainers.ContainerRequest{
+ Image: "starrocks/allin1-ubuntu:3.4.10",
+ ExposedPorts: []string{"9030/tcp"},
+ WaitingFor: wait.ForListeningPort("9030/tcp").WithStartupTimeout(5 * time.Minute),
+ }
+ c, err := testcontainers.GenericContainer(ctx, testcontainers.GenericContainerRequest{
+ ContainerRequest: req,
+ Started: true,
+ })
+ if err != nil {
+ return nil, err
+ }
+ host, err := c.Host(ctx)
+ if err != nil {
+ return nil, err
+ }
+ port, err := c.MappedPort(ctx, "9030/tcp")
+ if err != nil {
+ return nil, err
+ }
+ dsn := fmt.Sprintf("root@tcp(%s:%s)/?multiStatements=true", host, port.Port())
+ db, err := sql.Open("mysql", dsn)
+ if err != nil {
+ return nil, err
+ }
+ defer func() {
+ if retErr != nil {
+ db.Close()
+ }
+ }()
+ if err := waitStarRocksReady(ctx, db); err != nil {
+ return nil, err
+ }
+ return &Container{
+ container: c,
+ host: host,
+ port: port.Port(),
+ db: db,
+ }, nil
+}
+
+// GetTestStarRocksContainer creates a StarRocks container and fails the test on error. It
+// skips on non-amd64 hosts: the StarRocks all-in-one BE has no working arm64 build (it never
+// comes alive under emulation), so the readiness wait would otherwise time out.
+func GetTestStarRocksContainer(ctx context.Context, t testing.TB) *Container {
+ t.Helper()
+ if runtime.GOARCH != "amd64" {
+ t.Skipf("StarRocks requires an amd64 host; the all-in-one BE has no working arm64 build (GOARCH=%s)", runtime.GOARCH)
+ }
+ container, err := GetStarRocksContainer(ctx)
+ if err != nil {
+ t.Fatalf("failed to create StarRocks container: %v", err)
+ }
+ return container
+}
+
+// waitStarRocksReady blocks until at least one StarRocks backend is registered and alive,
+// which is required before tablet-allocating DDL (CREATE TABLE) succeeds. The check is
+// read-only on purpose: issuing DDL during allin1's first-boot disrupts backend
+// registration, so poll SHOW BACKENDS rather than probing with CREATE TABLE.
+func waitStarRocksReady(ctx context.Context, db *sql.DB) error {
+ ticker := time.NewTicker(3 * time.Second)
+ defer ticker.Stop()
+ timeout := time.After(10 * time.Minute)
+ for {
+ select {
+ case <-ticker.C:
+ if starRocksBackendAlive(ctx, db) {
+ return nil
+ }
+ case <-timeout:
+ return errors.Errorf("StarRocks backend did not become ready")
+ }
+ }
+}
+
+// starRocksBackendAlive reports whether SHOW BACKENDS lists a backend with Alive=true.
+func starRocksBackendAlive(ctx context.Context, db *sql.DB) bool {
+ rows, err := db.QueryContext(ctx, "SHOW BACKENDS")
+ if err != nil {
+ return false
+ }
+ defer rows.Close()
+ cols, err := rows.Columns()
+ if err != nil {
+ return false
+ }
+ aliveIdx := -1
+ for i, c := range cols {
+ if strings.EqualFold(c, "Alive") {
+ aliveIdx = i
+ break
+ }
+ }
+ if aliveIdx < 0 {
+ return false
+ }
+ alive := false
+ for rows.Next() {
+ vals := make([]sql.RawBytes, len(cols))
+ ptrs := make([]any, len(cols))
+ for i := range vals {
+ ptrs[i] = &vals[i]
+ }
+ if err := rows.Scan(ptrs...); err != nil {
+ return false
+ }
+ if strings.EqualFold(string(vals[aliveIdx]), "true") {
+ alive = true
+ break
+ }
+ }
+ if err := rows.Err(); err != nil {
+ return false
+ }
+ return alive
+}
+
// GetMSSQLContainer creates a Microsoft SQL Server container for testing
func GetMSSQLContainer(ctx context.Context) (retC *Container, retErr error) {
req := testcontainers.ContainerRequest{
diff --git a/backend/plugin/db/starrocks/dump.go b/backend/plugin/db/starrocks/dump.go
index 2365d841b3..5b83480766 100644
--- a/backend/plugin/db/starrocks/dump.go
+++ b/backend/plugin/db/starrocks/dump.go
@@ -98,8 +98,13 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *storepb.DatabaseSchem
// Disable foreign key check.
// mysqldump uses the same mechanism. When there is any schema or data dependency, we have to disable
// the unique and foreign key check so that the restoring will not fail.
- if _, err := io.WriteString(out, disableUniqueAndForeignKeyCheckStmt); err != nil {
- return err
+ // StarRocks has no MySQL UNIQUE_CHECKS/FOREIGN_KEY_CHECKS session variables (and does not
+ // enforce foreign keys), so emitting the guard makes the dump fail to replay with
+ // "Unknown system variable 'UNIQUE_CHECKS'".
+ if d.dbType != storepb.Engine_STARROCKS {
+ if _, err := io.WriteString(out, disableUniqueAndForeignKeyCheckStmt); err != nil {
+ return err
+ }
}
// Table and view statement.
@@ -150,18 +155,15 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *storepb.DatabaseSchem
return err
}
}
- // Construct final views and materialized views.
- for _, tbl := range tables {
- if tbl.TableType != viewTableType && tbl.TableType != materializedViewType {
- continue
- }
- // The temporary view just created above were used to satisfy the schema dependency. See comment above.
- // We have to drop the temporary and incorrect view here to recreate the final and correct one.
- dropStmt := "DROP VIEW IF EXISTS `%s`;\n"
- if tbl.TableType == materializedViewType {
- dropStmt = "DROP MATERIALIZED VIEW IF EXISTS `%s`;\n"
- }
- if _, err := io.WriteString(out, fmt.Sprintf(dropStmt, tbl.Name)); err != nil {
+ // Construct final views, then materialized views (see finalEmitOrder: an MV is
+ // materialized from its sources at CREATE time, so its source views must be real
+ // first, not the temporary SELECT 1 placeholders).
+ for _, tbl := range finalEmitOrder(tables) {
+ // The temporary placeholder created above is always a regular view (even for
+ // materialized views — see getTemporaryMaterializedView), so it must be dropped
+ // with DROP VIEW. DROP MATERIALIZED VIEW would not match the placeholder, leaving
+ // it in the shared namespace so the real CREATE MATERIALIZED VIEW below collides.
+ if _, err := io.WriteString(out, dropPlaceholderStmt(tbl.Name)); err != nil {
return err
}
if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.Statement)); err != nil {
@@ -191,14 +193,49 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *storepb.DatabaseSchem
}
}
- // Restore foreign key check.
- if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {
- return err
+ // Restore foreign key check (skipped for StarRocks, which lacks these variables).
+ if d.dbType != storepb.Engine_STARROCKS {
+ if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {
+ return err
+ }
}
return nil
}
+// finalEmitOrder returns the views and materialized views in the order their real
+// definitions must be emitted: all regular views first, then materialized views, each
+// group keeping its original relative order. information_schema.tables is unordered, and
+// a materialized view can be defined on a regular view and is materialized from its
+// sources at CREATE time, so its source views must already be real (not the temporary
+// SELECT 1 placeholders). Base tables are emitted separately and are excluded here.
+//
+// MVs keep their incoming order relative to each other, so a materialized view built on
+// another materialized view (nested async MVs) is not yet dependency-ordered and could
+// restore inactive if listed before its parent — tracked in BYT-9718.
+func finalEmitOrder(tables []*TableSchema) []*TableSchema {
+ var views, materializedViews []*TableSchema
+ for _, tbl := range tables {
+ switch tbl.TableType {
+ case viewTableType:
+ views = append(views, tbl)
+ case materializedViewType:
+ materializedViews = append(materializedViews, tbl)
+ default:
+ // Base tables and any other types are emitted elsewhere; skip here.
+ }
+ }
+ return append(views, materializedViews...)
+}
+
+// dropPlaceholderStmt returns the statement that drops the temporary placeholder emitted
+// for a view or materialized view before its real definition. The placeholder is always a
+// regular view (see getTemporaryMaterializedView), so it is dropped with DROP VIEW
+// regardless of the final object type.
+func dropPlaceholderStmt(name string) string {
+ return fmt.Sprintf("DROP VIEW IF EXISTS `%s`;\n", name)
+}
+
func getTemporaryView(name string, columns []string) string {
var parts []string
for _, col := range columns {
@@ -276,35 +313,13 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema
return nil, err
}
- // For Doris, get materialized views using mv_infos() and mark them appropriately
- materializedViewNames := make(map[string]bool)
- if dbType == storepb.Engine_DORIS {
- mvQuery := fmt.Sprintf(`SELECT Name FROM mv_infos("database"="%s")`, dbName)
- mvRows, err := db.Query(mvQuery)
- if err != nil {
- // mv_infos() might not be available in older versions, just log and continue
- slog.Debug("failed to query mv_infos(), might not be supported", slog.String("error", err.Error()))
- } else {
- defer mvRows.Close()
- for mvRows.Next() {
- var mvName string
- if err := mvRows.Scan(&mvName); err != nil {
- return nil, err
- }
- materializedViewNames[mvName] = true
- }
- if err := mvRows.Err(); err != nil {
- return nil, err
- }
- }
- }
-
- // Update table types for materialized views that are marked as BASE TABLE
- for _, tbl := range tables {
- if tbl.TableType == baseTableType && materializedViewNames[tbl.Name] {
- tbl.TableType = materializedViewType
- }
+ // Materialized views need a separate catalog lookup and re-tagging so the dump emits
+ // SHOW CREATE MATERIALIZED VIEW for them (Doris reports them as BASE TABLE, StarRocks as VIEW).
+ materializedViewNames, err := getMaterializedViewNames(db, dbName, dbType)
+ if err != nil {
+ return nil, err
}
+ markMaterializedViews(tables, materializedViewNames)
var result []*TableSchema
for _, tbl := range tables {
@@ -334,6 +349,69 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema
return result, nil
}
+// getMaterializedViewNames returns the set of materialized-view names in the database.
+// Doris and StarRocks expose them through different catalogs; the query is best-effort
+// (an unsupported engine/version logs and yields an empty set). StarRocks sync rollups
+// (REFRESH_TYPE='ROLLUP') are excluded — they are rollup indexes on the base table, not
+// standalone views.
+func getMaterializedViewNames(db *sql.DB, dbName string, dbType storepb.Engine) (map[string]bool, error) {
+ names := make(map[string]bool)
+
+ var query string
+ switch dbType {
+ case storepb.Engine_DORIS:
+ query = fmt.Sprintf(`SELECT Name FROM mv_infos("database"="%s")`, dbName)
+ case storepb.Engine_STARROCKS:
+ // IFNULL guards the bare-string scan below: database/sql errors on a NULL->string scan.
+ query = fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, dbName)
+ default:
+ return names, nil
+ }
+
+ rows, err := db.Query(query)
+ if err != nil {
+ // The catalog may be unavailable on older engine versions; log and continue.
+ slog.Debug("failed to query materialized views, might not be supported", slog.String("error", err.Error()))
+ return names, nil
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var name string
+ if dbType == storepb.Engine_STARROCKS {
+ var refreshType string
+ if err := rows.Scan(&name, &refreshType); err != nil {
+ return nil, err
+ }
+ if isSyncRollup(refreshType) {
+ continue
+ }
+ } else if err := rows.Scan(&name); err != nil {
+ return nil, err
+ }
+ names[name] = true
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return names, nil
+}
+
+// markMaterializedViews re-tags rows that are actually materialized views so the dump
+// emits SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE' and
+// StarRocks as 'VIEW'; either becomes 'MATERIALIZED VIEW' when its name is in the
+// materialized-view set. Regular tables and views are left untouched.
+func markMaterializedViews(tables []*TableSchema, materializedViewNames map[string]bool) {
+ for _, tbl := range tables {
+ if !materializedViewNames[tbl.Name] {
+ continue
+ }
+ if tbl.TableType == baseTableType || tbl.TableType == viewTableType {
+ tbl.TableType = materializedViewType
+ }
+ }
+}
+
// getTableStmt gets the create statement of a table.
func getTableStmt(db *sql.DB, dbName, tblName, tblType string) (string, error) {
switch tblType {
diff --git a/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go b/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go
new file mode 100644
index 0000000000..1cf3091538
--- /dev/null
+++ b/backend/plugin/db/starrocks/dump_e2e_testcontainer_test.go
@@ -0,0 +1,89 @@
+package starrocks
+
+import (
+ "bytes"
+ "context"
+ "database/sql"
+ "fmt"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+// TestDump_StarRocksMaterializedViewRoundTrip locks the two dump fixes (BYT-9689) against a
+// real StarRocks (amd64-only; skipped in -short). A materialized view defined on a regular
+// view is dumped, then replayed into a fresh database of the same name, and must come back:
+// - without a name collision (P1: the temporary placeholder is dropped as a view, so the
+// real CREATE MATERIALIZED VIEW does not collide with a leftover placeholder), and
+// - IS_ACTIVE=true (P2: the MV is emitted after its source view, so it is created against
+// the real view, not inactivated when a placeholder is dropped under it).
+//
+// The dump is schema-only (--no-data), so IS_ACTIVE — not row data — is the P2 signal.
+func TestDump_StarRocksMaterializedViewRoundTrip(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skipping StarRocks testcontainer test in short mode")
+ }
+ ctx := context.Background()
+ c := testcontainer.GetTestStarRocksContainer(ctx, t)
+ defer c.Close(ctx)
+ db := c.GetDB()
+
+ for _, stmt := range []string{
+ "CREATE DATABASE rt",
+ "CREATE TABLE rt.t (id INT, amt INT) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')",
+ "INSERT INTO rt.t VALUES (1,10),(2,20)",
+ "CREATE VIEW rt.vw AS SELECT id, amt FROM rt.t",
+ "CREATE MATERIALIZED VIEW rt.mv REFRESH ASYNC AS SELECT id, sum(amt) AS total FROM rt.vw GROUP BY id",
+ } {
+ if _, err := db.ExecContext(ctx, stmt); err != nil {
+ t.Fatalf("setup failed: %s: %v", stmt, err)
+ }
+ }
+ requireMVActive(ctx, t, db, "rt", "mv")
+
+ // Produce the dump through the real driver.
+ d := &Driver{dbType: storepb.Engine_STARROCKS, db: db, databaseName: "rt"}
+ var buf bytes.Buffer
+ require.NoError(t, d.Dump(ctx, &buf, nil))
+ dump := buf.String()
+ require.Contains(t, dump, "CREATE MATERIALIZED VIEW", "dump should emit the MV")
+
+ // Restore into a fresh same-name database. StarRocks qualifies view/MV references with
+ // the original db, so the restore target must reuse the name. DROP+CREATE+USE+replay run
+ // in one multi-statement Exec so USE applies to the unqualified statements on the same
+ // pooled connection.
+ restore := "DROP DATABASE IF EXISTS rt;\nCREATE DATABASE rt;\nUSE rt;\n" + dump
+ if _, err := db.ExecContext(ctx, restore); err != nil {
+ t.Fatalf("dump replay failed (P1 collision or unreplayable DDL): %v", err)
+ }
+
+ // P2: the restored MV must be active, not inactivated by the placeholder dance.
+ requireMVActive(ctx, t, db, "rt", "mv")
+
+ // And it re-syncs cleanly: mv as a materialized view, no stale placeholder.
+ meta, err := d.SyncDBSchema(ctx)
+ require.NoError(t, err)
+ require.Len(t, meta.Schemas, 1)
+ require.Len(t, meta.Schemas[0].MaterializedViews, 1)
+ require.Equal(t, "mv", meta.Schemas[0].MaterializedViews[0].Name)
+}
+
+// requireMVActive waits until the named materialized view reports IS_ACTIVE=true.
+func requireMVActive(ctx context.Context, t *testing.T, db *sql.DB, schema, name string) {
+ t.Helper()
+ q := fmt.Sprintf("SELECT IS_ACTIVE FROM information_schema.materialized_views WHERE TABLE_SCHEMA='%s' AND TABLE_NAME='%s'", schema, name)
+ require.Eventually(t, func() bool {
+ var active string
+ if err := db.QueryRowContext(ctx, q).Scan(&active); err != nil {
+ return false
+ }
+ return active == "true"
+ // information_schema.materialized_views (listMaterializedViewStatus, an FE->BE RPC)
+ // is unavailable for ~tens of seconds after a fresh allin1 boot ("BE host unknown")
+ // before it settles, so allow a generous window.
+ }, 180*time.Second, 3*time.Second, "MV %s.%s did not become active", schema, name)
+}
diff --git a/backend/plugin/db/starrocks/dump_materialized_view_test.go b/backend/plugin/db/starrocks/dump_materialized_view_test.go
new file mode 100644
index 0000000000..34a31ed117
--- /dev/null
+++ b/backend/plugin/db/starrocks/dump_materialized_view_test.go
@@ -0,0 +1,60 @@
+package starrocks
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+// BYT-9689: the dump path re-tags materialized-view rows so it emits
+// SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE',
+// StarRocks as 'VIEW'; both must become 'MATERIALIZED VIEW' when the name is a known
+// MV, while regular tables and views are left untouched.
+func TestMarkMaterializedViews(t *testing.T) {
+ tables := []*TableSchema{
+ {Name: "mv_doris", TableType: baseTableType}, // Doris MV reported as BASE TABLE
+ {Name: "mv_starrocks", TableType: viewTableType}, // StarRocks MV reported as VIEW
+ {Name: "v_regular", TableType: viewTableType}, // regular view (not an MV)
+ {Name: "t_plain", TableType: baseTableType}, // plain table
+ }
+ markMaterializedViews(tables, map[string]bool{"mv_doris": true, "mv_starrocks": true})
+
+ require.Equal(t, materializedViewType, tables[0].TableType)
+ require.Equal(t, materializedViewType, tables[1].TableType)
+ require.Equal(t, viewTableType, tables[2].TableType, "regular view must stay a view")
+ require.Equal(t, baseTableType, tables[3].TableType, "plain table must stay a table")
+}
+
+// BYT-9689: the dump emits a temporary regular-view placeholder for every view AND
+// materialized view (getTemporaryMaterializedView creates a CREATE VIEW), then drops it
+// before emitting the real definition. The drop must therefore be DROP VIEW even for an
+// MV — DROP MATERIALIZED VIEW would not match the placeholder, leaving it in the shared
+// namespace so the real CREATE MATERIALIZED VIEW collides with it on replay.
+func TestMaterializedViewPlaceholderIsDroppedAsView(t *testing.T) {
+ placeholder := getTemporaryMaterializedView("mv1", []string{"a"})
+ require.Contains(t, placeholder, "CREATE VIEW `mv1`")
+ require.NotContains(t, placeholder, "CREATE MATERIALIZED VIEW")
+
+ require.Equal(t, "DROP VIEW IF EXISTS `mv1`;\n", dropPlaceholderStmt("mv1"))
+}
+
+// BYT-9689: information_schema.tables is unordered, so an MV can be listed before a view
+// it is defined on. A materialized view is materialized from its sources at CREATE time
+// (unlike a lazy regular view), so all regular views must be emitted — and on restore,
+// created real — before any materialized view; otherwise the MV would materialize from
+// the temporary SELECT 1 placeholder. finalEmitOrder partitions views ahead of MVs while
+// preserving each group's relative order, and drops base tables (emitted elsewhere).
+func TestFinalEmitOrder_ViewsBeforeMaterializedViews(t *testing.T) {
+ tables := []*TableSchema{
+ {Name: "mv1", TableType: materializedViewType},
+ {Name: "v1", TableType: viewTableType},
+ {Name: "t1", TableType: baseTableType},
+ {Name: "mv2", TableType: materializedViewType},
+ {Name: "v2", TableType: viewTableType},
+ }
+ var names []string
+ for _, tbl := range finalEmitOrder(tables) {
+ names = append(names, tbl.Name)
+ }
+ require.Equal(t, []string{"v1", "v2", "mv1", "mv2"}, names)
+}
diff --git a/backend/plugin/db/starrocks/sync.go b/backend/plugin/db/starrocks/sync.go
index bcc15dcf5a..f0b8db4c98 100644
--- a/backend/plugin/db/starrocks/sync.go
+++ b/backend/plugin/db/starrocks/sync.go
@@ -114,6 +114,85 @@ func (d *Driver) getServerVariable(ctx context.Context, varName string) (string,
return value, nil
}
+// isSyncRollup reports whether a StarRocks information_schema.materialized_views row
+// describes a synchronous rollup rather than a standalone (async) materialized view.
+// Sync rollups are rollup indexes on the base table: they appear in that catalog with
+// REFRESH_TYPE='ROLLUP' but have no information_schema.tables row, so they are excluded
+// from the materialized-view set. Only the confirmed 'ROLLUP' value is excluded — async
+// refresh types and any unknown/empty value are kept.
+func isSyncRollup(refreshType string) bool {
+ return refreshType == "ROLLUP"
+}
+
+// isMaterializedView reports whether a table row is a materialized view, based on its
+// table type and membership in the materialized-view set. StarRocks reports
+// materialized views with TABLE_TYPE='VIEW', Doris with 'BASE TABLE', and some setups
+// with 'MATERIALIZED VIEW'; in every case, presence in materializedViewMap is the
+// authoritative signal.
+func isMaterializedView(tableType string, key db.TableKey, materializedViewMap map[db.TableKey]*storepb.MaterializedViewMetadata) bool {
+ if _, ok := materializedViewMap[key]; !ok {
+ return false
+ }
+ switch tableType {
+ case baseTableType, viewTableType, materializedViewType:
+ return true
+ default:
+ return false
+ }
+}
+
+// getMaterializedViews returns the materialized views of the current database keyed by
+// table name. Doris and StarRocks expose them through different catalogs, so the query
+// and projection are engine-specific. The query is best-effort: an engine/version that
+// does not support it is logged and yields an empty set rather than failing the sync.
+func (d *Driver) getMaterializedViews(ctx context.Context) (map[db.TableKey]*storepb.MaterializedViewMetadata, error) {
+ materializedViewMap := make(map[db.TableKey]*storepb.MaterializedViewMetadata)
+
+ var query string
+ switch d.dbType {
+ case storepb.Engine_DORIS:
+ // Doris exposes materialized views via the mv_infos() table-valued function.
+ // https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/mv_infos
+ query = fmt.Sprintf(`SELECT Name, QuerySql FROM mv_infos("database"="%s")`, d.databaseName)
+ case storepb.Engine_STARROCKS:
+ // StarRocks exposes materialized views via information_schema.materialized_views.
+ // REFRESH_TYPE separates async MVs from synchronous rollups, which are excluded.
+ // IFNULL guards the bare-string scans below: database/sql errors on a NULL->string scan.
+ query = fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, ''), IFNULL(MATERIALIZED_VIEW_DEFINITION, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, d.databaseName)
+ default:
+ return materializedViewMap, nil
+ }
+
+ rows, err := d.db.QueryContext(ctx, query)
+ if err != nil {
+ // The catalog may be unavailable on older engine versions; log and continue.
+ slog.Debug("failed to query materialized views, might not be supported in this version", log.BBError(err))
+ return materializedViewMap, nil
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ materializedView := &storepb.MaterializedViewMetadata{}
+ if d.dbType == storepb.Engine_STARROCKS {
+ var refreshType string
+ if err := rows.Scan(&materializedView.Name, &refreshType, &materializedView.Definition); err != nil {
+ return nil, err
+ }
+ if isSyncRollup(refreshType) {
+ continue
+ }
+ } else if err := rows.Scan(&materializedView.Name, &materializedView.Definition); err != nil {
+ return nil, err
+ }
+ key := db.TableKey{Schema: "", Table: materializedView.Name}
+ materializedViewMap[key] = materializedView
+ }
+ if err := rows.Err(); err != nil {
+ return nil, util.FormatErrorWithQuery(err, query)
+ }
+ return materializedViewMap, nil
+}
+
// SyncDBSchema syncs a single database schema.
func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetadata, error) {
schemaMetadata := &storepb.SchemaMetadata{
@@ -226,39 +305,10 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetad
return nil, util.FormatErrorWithQuery(err, viewQuery)
}
- // Query materialized view info.
- // For Doris, we use the mv_infos() table-valued function to get materialized view information.
- // Sync materialized views are part of the base table and don't need special handling.
- materializedViewMap := make(map[db.TableKey]*storepb.MaterializedViewMetadata)
-
- // Only query materialized views for Doris
- if d.dbType == storepb.Engine_DORIS {
- // Use mv_infos() table-valued function to get materialized view information
- // https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/mv_infos
- materializedViewQuery := fmt.Sprintf(`SELECT Name, QuerySql FROM mv_infos("database"="%s")`, d.databaseName)
- materializedViewRows, err := d.db.QueryContext(ctx, materializedViewQuery)
- if err != nil {
- // mv_infos() might not be available in older Doris versions, log and continue
- slog.Debug("failed to query materialized views using mv_infos(), might not be supported in this version", log.BBError(err))
- } else {
- defer materializedViewRows.Close()
- for materializedViewRows.Next() {
- materializedView := &storepb.MaterializedViewMetadata{}
- if err := materializedViewRows.Scan(
- &materializedView.Name,
- &materializedView.Definition,
- ); err != nil {
- return nil, err
- }
- key := db.TableKey{Schema: "", Table: materializedView.Name}
- materializedViewMap[key] = materializedView
- slog.Debug("found materialized view", slog.String("name", materializedView.Name), slog.String("database", d.databaseName))
- }
- if err := materializedViewRows.Err(); err != nil {
- return nil, util.FormatErrorWithQuery(err, materializedViewQuery)
- }
- slog.Debug("total materialized views found", slog.Int("count", len(materializedViewMap)), slog.String("database", d.databaseName))
- }
+ // Query materialized view info (engine-specific catalog; best-effort).
+ materializedViewMap, err := d.getMaterializedViews(ctx)
+ if err != nil {
+ return nil, err
}
// Query table info.
@@ -303,48 +353,42 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*storepb.DatabaseSchemaMetad
}
key := db.TableKey{Schema: "", Table: tableName}
+ // StarRocks reports materialized views as TABLE_TYPE='VIEW', Doris as 'BASE TABLE';
+ // route any row whose name is in the materialized-view set to MaterializedViews.
+ if isMaterializedView(tableType, key, materializedViewMap) {
+ materializedView := materializedViewMap[key]
+ materializedView.Comment = comment
+ schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, materializedView)
+ continue
+ }
switch tableType {
case baseTableType:
- // Check if this is actually a materialized view (for Doris)
- // In Doris, materialized views appear as BASE TABLE in information_schema.TABLES
- if materializedView, ok := materializedViewMap[key]; ok {
- slog.Debug("identified BASE TABLE as materialized view", slog.String("name", tableName), slog.String("database", d.databaseName))
- materializedView.Comment = comment
- schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, materializedView)
- } else {
- slog.Debug("adding BASE TABLE as regular table", slog.String("name", tableName), slog.String("database", d.databaseName))
- tableMetadata := &storepb.TableMetadata{
- Name: tableName,
- Columns: columnMap[key],
- Engine: engine,
- Collation: collation,
- RowCount: rowCount,
- DataSize: dataSize,
- IndexSize: indexSize,
- DataFree: dataFree,
- CreateOptions: createOptions,
- Comment: comment,
- }
- if tableCollation.Valid {
- tableMetadata.Collation = tableCollation.String
- }
- // TODO(d): add index information whenever it is available.
- schemaMetadata.Tables = append(schemaMetadata.Tables, tableMetadata)
+ tableMetadata := &storepb.TableMetadata{
+ Name: tableName,
+ Columns: columnMap[key],
+ Engine: engine,
+ Collation: collation,
+ RowCount: rowCount,
+ DataSize: dataSize,
+ IndexSize: indexSize,
+ DataFree: dataFree,
+ CreateOptions: createOptions,
+ Comment: comment,
}
+ if tableCollation.Valid {
+ tableMetadata.Collation = tableCollation.String
+ }
+ // TODO(d): add index information whenever it is available.
+ schemaMetadata.Tables = append(schemaMetadata.Tables, tableMetadata)
case viewTableType:
if view, ok := viewMap[key]; ok {
view.Comment = comment
schemaMetadata.Views = append(schemaMetadata.Views, view)
}
- case materializedViewType:
- // For databases that properly set TABLE_TYPE = 'MATERIALIZED VIEW'
- if materializedView, ok := materializedViewMap[key]; ok {
- materializedView.Comment = comment
- schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, materializedView)
- }
default:
- // Skip unknown table types (e.g., SYSTEM VIEW, TEMPORARY, etc.)
- slog.Debug("skipping unknown table type", slog.String("tableName", tableName), slog.String("tableType", tableType))
+ // Includes 'MATERIALIZED VIEW' rows absent from the materialized-view set
+ // and any other unrecognized type.
+ slog.Debug("skipping unhandled table type", slog.String("tableName", tableName), slog.String("tableType", tableType))
}
}
if err := tableRows.Err(); err != nil {
diff --git a/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go b/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go
new file mode 100644
index 0000000000..ae004487d9
--- /dev/null
+++ b/backend/plugin/db/starrocks/sync_e2e_testcontainer_test.go
@@ -0,0 +1,75 @@
+package starrocks
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/common/testcontainer"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+// TestSyncDBSchema_StarRocksMaterializedView_E2E brings up a real StarRocks (amd64-only;
+// runs in CI, skipped in -short) and verifies BYT-9689 end to end: an async materialized
+// view is synced under MaterializedViews with its definition, while a synchronous rollup
+// (REFRESH_TYPE='ROLLUP', which has no information_schema.tables row) is excluded.
+func TestSyncDBSchema_StarRocksMaterializedView_E2E(t *testing.T) {
+ if testing.Short() {
+ t.Skip("skipping StarRocks testcontainer test in short mode")
+ }
+ ctx := context.Background()
+ c := testcontainer.GetTestStarRocksContainer(ctx, t)
+ defer c.Close(ctx)
+ db := c.GetDB()
+
+ for _, stmt := range []string{
+ "CREATE DATABASE mvtest",
+ "CREATE TABLE mvtest.sales (id BIGINT, amt INT) DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES('replication_num'='1')",
+ "INSERT INTO mvtest.sales VALUES (1,10),(2,20)",
+ "CREATE MATERIALIZED VIEW mvtest.mv_async REFRESH ASYNC AS SELECT id, sum(amt) AS total FROM mvtest.sales GROUP BY id",
+ "CREATE MATERIALIZED VIEW mvtest.mv_rollup AS SELECT id, sum(amt) FROM mvtest.sales GROUP BY id",
+ } {
+ if _, err := db.ExecContext(ctx, stmt); err != nil {
+ t.Fatalf("setup failed: %s: %v", stmt, err)
+ }
+ }
+
+ // The sync rollup builds asynchronously; wait until it registers in
+ // information_schema.materialized_views so the test deterministically exercises the
+ // rollup-exclusion path (it appears there as REFRESH_TYPE='ROLLUP' but never as a
+ // information_schema.tables row).
+ require.Eventually(t, func() bool {
+ var n int
+ if err := db.QueryRowContext(ctx,
+ "SELECT COUNT(*) FROM information_schema.materialized_views WHERE TABLE_SCHEMA='mvtest' AND REFRESH_TYPE='ROLLUP'").Scan(&n); err != nil {
+ return false
+ }
+ return n == 1
+ // information_schema.materialized_views (an FE->BE RPC) is unavailable for tens of
+ // seconds after a fresh allin1 boot before it settles; allow a generous window.
+ }, 150*time.Second, 3*time.Second, "sync rollup did not register in materialized_views")
+
+ d := &Driver{dbType: storepb.Engine_STARROCKS, db: db, databaseName: "mvtest"}
+ meta, err := d.SyncDBSchema(ctx)
+ require.NoError(t, err)
+ require.Len(t, meta.Schemas, 1)
+ schema := meta.Schemas[0]
+
+ // The async MV is synced under MaterializedViews with its definition.
+ require.Len(t, schema.MaterializedViews, 1, "exactly the async MV must be synced")
+ require.Equal(t, "mv_async", schema.MaterializedViews[0].Name)
+ require.Contains(t, schema.MaterializedViews[0].Definition, "CREATE MATERIALIZED VIEW")
+
+ // The base table is synced as a table; the sync rollup leaks nowhere; the MV is not
+ // misfiled as a regular view.
+ require.Len(t, schema.Tables, 1)
+ require.Equal(t, "sales", schema.Tables[0].Name)
+ for _, mv := range schema.MaterializedViews {
+ require.NotEqual(t, "mv_rollup", mv.Name, "sync rollup must be excluded from MaterializedViews")
+ }
+ for _, v := range schema.Views {
+ require.NotEqual(t, "mv_async", v.Name, "MV must not be misfiled as a regular view")
+ }
+}
diff --git a/backend/plugin/db/starrocks/sync_materialized_view_test.go b/backend/plugin/db/starrocks/sync_materialized_view_test.go
new file mode 100644
index 0000000000..c8fa683e9e
--- /dev/null
+++ b/backend/plugin/db/starrocks/sync_materialized_view_test.go
@@ -0,0 +1,45 @@
+package starrocks
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+)
+
+// BYT-9689: StarRocks reports materialized views in information_schema.tables with
+// TABLE_TYPE='VIEW' and omits them from information_schema.views (verified on a live
+// 3.4.10 BE-alive smoke test), while Doris reports them as 'BASE TABLE'. The shared
+// driver previously only matched Doris's 'BASE TABLE' rows, so StarRocks MVs were
+// dropped. Membership in the materialized-view set is the authoritative signal.
+func TestIsMaterializedView(t *testing.T) {
+ mvMap := map[db.TableKey]*storepb.MaterializedViewMetadata{
+ {Schema: "", Table: "mv_async"}: {Name: "mv_async"},
+ }
+ mvKey := db.TableKey{Schema: "", Table: "mv_async"}
+
+ // StarRocks reports the MV as VIEW; Doris as BASE TABLE; both map to an MV.
+ require.True(t, isMaterializedView(viewTableType, mvKey, mvMap))
+ require.True(t, isMaterializedView(baseTableType, mvKey, mvMap))
+ require.True(t, isMaterializedView(materializedViewType, mvKey, mvMap))
+
+ // A name absent from the MV set is never a materialized view.
+ require.False(t, isMaterializedView(viewTableType, db.TableKey{Table: "v_regular"}, mvMap))
+ require.False(t, isMaterializedView(baseTableType, db.TableKey{Table: "sales"}, mvMap))
+}
+
+// BYT-9689: StarRocks synchronous rollups are listed in
+// information_schema.materialized_views with REFRESH_TYPE='ROLLUP' but, unlike async
+// MVs, have no information_schema.tables row (they are rollup indexes on the base
+// table). Only confirmed rollups are excluded; async MVs (ASYNC/MANUAL/...) are kept, so
+// a future refresh type isn't silently dropped. The query coalesces SQL NULL to the
+// empty string via IFNULL (database/sql cannot scan NULL into a string), so a NULL
+// refresh type reaches this check as the empty string, which must also be kept.
+func TestIsSyncRollup(t *testing.T) {
+ require.True(t, isSyncRollup("ROLLUP"))
+ require.False(t, isSyncRollup("ASYNC"))
+ require.False(t, isSyncRollup("MANUAL"))
+ require.False(t, isSyncRollup("")) // IFNULL-coalesced NULL or a genuinely empty type.
+}