Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

refactor(doris,starrocks): split the engines, and fix their changed-resource extraction (#21731)

Danny Xu07c8fc2a8b68
diff --git a/backend/plugin/db/doris/doris.go b/backend/plugin/db/doris/doris.go
new file mode 100644
index 0000000000..52449f7a3e
--- /dev/null
+++ b/backend/plugin/db/doris/doris.go
@@ -0,0 +1,398 @@
+// Package doris is the plugin for the Apache Doris driver.
+package doris
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "log/slog"
+ "net"
+ "regexp"
+ "strings"
+ "time"
+
+ dorisomni "github.com/bytebase/omni/doris/parser"
+ "github.com/go-sql-driver/mysql"
+ "github.com/google/uuid"
+ "github.com/pkg/errors"
+ "go.uber.org/multierr"
+ "golang.org/x/crypto/ssh"
+ "google.golang.org/protobuf/types/known/durationpb"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+ "github.com/bytebase/bytebase/backend/plugin/db/transaction"
+ "github.com/bytebase/bytebase/backend/plugin/db/util"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+
+ // Register how Doris plans a statement, for base.ExplainStatement below.
+ _ "github.com/bytebase/bytebase/backend/plugin/parser/doris"
+)
+
+var (
+ baseTableType = "BASE TABLE"
+ viewTableType = "VIEW"
+ materializedViewType = "MATERIALIZED VIEW"
+
+ _ db.Driver = (*Driver)(nil)
+)
+
+func init() {
+ db.Register(storepb.Engine_DORIS, newDriver)
+}
+
+// Driver is the Doris driver.
+type Driver struct {
+ dbType storepb.Engine
+ db *sql.DB
+ databaseName string
+ sshClient *ssh.Client
+
+ // Called upon driver.Open() finishes.
+ openCleanUp []func()
+}
+
+func newDriver() db.Driver {
+ return &Driver{}
+}
+
+// Open opens a Doris driver.
+func (d *Driver) Open(_ context.Context, dbType storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {
+ defer func() {
+ for _, f := range d.openCleanUp {
+ f()
+ }
+ }()
+
+ protocol := "tcp"
+ if strings.HasPrefix(connCfg.DataSource.Host, "/") {
+ protocol = "unix"
+ }
+ params := []string{"multiStatements=true", "maxAllowedPacket=0", "interpolateParams=true"}
+ if connCfg.DataSource.GetSshHost() != "" {
+ sshClient, err := util.GetSSHClient(connCfg.DataSource)
+ if err != nil {
+ return nil, err
+ }
+ d.sshClient = sshClient
+ // Now we register the dialer with the ssh connection as a parameter.
+ protocol = "mysql-tcp-" + uuid.NewString()[:8]
+ // Now we register the dialer with the ssh connection as a parameter.
+ mysql.RegisterDialContext(protocol, func(_ context.Context, addr string) (net.Conn, error) {
+ return sshClient.Dial("tcp", addr)
+ })
+ }
+
+ tlscfg, err := util.GetTLSConfig(connCfg.DataSource)
+ if err != nil {
+ return nil, errors.Wrap(err, "sql: tls config error")
+ }
+ tlsKey := uuid.NewString()
+ if tlscfg != nil {
+ if err := mysql.RegisterTLSConfig(tlsKey, tlscfg); err != nil {
+ return nil, errors.Wrap(err, "sql: failed to register tls config")
+ }
+ // TLS config is only used during sql.Open, so should be safe to deregister afterwards.
+ d.openCleanUp = append(d.openCleanUp, func() { mysql.DeregisterTLSConfig(tlsKey) })
+ params = append(params, fmt.Sprintf("tls=%s", tlsKey))
+ }
+
+ dsn := fmt.Sprintf("%s:%s@%s(%s:%s)/%s?%s", connCfg.DataSource.Username, connCfg.Password, protocol, connCfg.DataSource.Host, connCfg.DataSource.Port, connCfg.ConnectionContext.DatabaseName, strings.Join(params, "&"))
+ db, err := sql.Open("mysql", dsn)
+ if err != nil {
+ return nil, err
+ }
+ d.dbType = dbType
+ d.db = db
+ // TODO(d): remove the work-around once we have clean-up the migration connection hack.
+ db.SetConnMaxLifetime(2 * time.Hour)
+ db.SetMaxOpenConns(50)
+ db.SetMaxIdleConns(15)
+ d.databaseName = connCfg.ConnectionContext.DatabaseName
+ return d, nil
+}
+
+// Close closes the driver.
+func (d *Driver) Close(context.Context) error {
+ var err error
+ err = multierr.Append(err, d.db.Close())
+ if d.sshClient != nil {
+ err = multierr.Append(err, d.sshClient.Close())
+ }
+ return err
+}
+
+// Ping pings the database.
+func (d *Driver) Ping(ctx context.Context) error {
+ return d.db.PingContext(ctx)
+}
+
+// GetDB gets the database.
+func (d *Driver) GetDB() *sql.DB {
+ return d.db
+}
+
+// getVersion gets the version.
+func (d *Driver) getVersion(ctx context.Context) (string, string, error) {
+ query := "SELECT VERSION()"
+ var version string
+ if err := d.db.QueryRowContext(ctx, query).Scan(&version); err != nil {
+ if err == sql.ErrNoRows {
+ return "", "", common.FormatDBErrorEmptyRowWithQuery(query)
+ }
+ return "", "", util.FormatErrorWithQuery(err, query)
+ }
+
+ return parseVersion(version)
+}
+
+func parseVersion(version string) (string, string, error) {
+ if loc := regexp.MustCompile(`^\d+.\d+.\d+`).FindStringIndex(version); loc != nil {
+ return version[loc[0]:loc[1]], version[loc[1]:], nil
+ }
+ return "", "", errors.Errorf("failed to parse version %q", version)
+}
+
+// Execute executes a SQL statement.
+func (d *Driver) Execute(ctx context.Context, statement string, opts db.ExecuteOptions) (int64, error) {
+ // Parse transaction mode from the script
+ config, cleanedStatement := base.ParseTransactionConfig(statement)
+ statement = cleanedStatement
+ transactionMode := config.Mode
+
+ // Apply default when transaction mode is not specified
+ if transactionMode == transaction.ModeUnspecified {
+ transactionMode = transaction.DefaultMode()
+ }
+
+ // Only preprocess when DELIMITER directives are present; normal
+ // Doris SQL passes through unchanged.
+ if containsDelimiterDirective(statement) {
+ segments := dorisomni.Split(statement)
+ var buf strings.Builder
+ for i, seg := range segments {
+ if i > 0 {
+ buf.WriteString(";\n")
+ }
+ buf.WriteString(seg.Text)
+ }
+ statement = buf.String()
+ }
+ // Execute based on transaction mode
+ // Note: Doris is an OLAP database with limited transaction support.
+ // For DDL operations, transactions are not supported. For DML operations,
+ // Doris supports transactions within certain limitations.
+ if transactionMode == transaction.ModeOff || opts.CreateDatabase {
+ return d.executeInAutoCommitMode(ctx, statement)
+ }
+ return d.executeInTransactionMode(ctx, statement, opts)
+}
+
+func (d *Driver) executeInTransactionMode(ctx context.Context, statement string, opts db.ExecuteOptions) (int64, error) {
+ conn, err := d.db.Conn(ctx)
+ if err != nil {
+ return 0, err
+ }
+ defer conn.Close()
+
+ connectionID, err := getConnectionID(ctx, conn)
+ if err != nil {
+ return 0, err
+ }
+ slog.Debug("connectionID", slog.String("connectionID", connectionID))
+
+ tx, err := conn.BeginTx(ctx, nil)
+ if err != nil {
+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_BEGIN, err.Error())
+ return 0, errors.Wrapf(err, "failed to begin execute transaction")
+ }
+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_BEGIN, "")
+
+ committed := false
+ defer func() {
+ if !committed {
+ err := tx.Rollback()
+ var rerr string
+ if err != nil && !errors.Is(err, sql.ErrTxDone) {
+ rerr = err.Error()
+ slog.Debug("failed to rollback transaction", log.BBError(err))
+ }
+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_ROLLBACK, rerr)
+ }
+ }()
+
+ sqlResult, err := tx.ExecContext(ctx, statement)
+ if err != nil {
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ slog.Info("cancel connection", slog.String("connectionID", connectionID))
+ if err := d.StopConnectionByID(connectionID); err != nil {
+ slog.Error("failed to cancel connection", slog.String("connectionID", connectionID), log.BBError(err))
+ }
+ }
+ return 0, err
+ }
+
+ rowsAffected, err := sqlResult.RowsAffected()
+ if err != nil {
+ // Since we cannot differentiate DDL and DML yet, we have to ignore the error.
+ slog.Debug("rowsAffected returns error", log.BBError(err))
+ }
+
+ if err := tx.Commit(); err != nil {
+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_COMMIT, err.Error())
+ return 0, errors.Wrapf(err, "failed to commit execute transaction")
+ }
+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_COMMIT, "")
+ committed = true
+ return rowsAffected, nil
+}
+
+func (d *Driver) executeInAutoCommitMode(ctx context.Context, statement string) (int64, error) {
+ conn, err := d.db.Conn(ctx)
+ if err != nil {
+ return 0, err
+ }
+ defer conn.Close()
+
+ connectionID, err := getConnectionID(ctx, conn)
+ if err != nil {
+ return 0, err
+ }
+ slog.Debug("connectionID", slog.String("connectionID", connectionID))
+
+ // Execute in auto-commit mode
+ sqlResult, err := conn.ExecContext(ctx, statement)
+ if err != nil {
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ slog.Info("cancel connection", slog.String("connectionID", connectionID))
+ if err := d.StopConnectionByID(connectionID); err != nil {
+ slog.Error("failed to cancel connection", slog.String("connectionID", connectionID), log.BBError(err))
+ }
+ }
+ return 0, err
+ }
+
+ rowsAffected, err := sqlResult.RowsAffected()
+ if err != nil {
+ // Since we cannot differentiate DDL and DML yet, we have to ignore the error.
+ slog.Debug("rowsAffected returns error", log.BBError(err))
+ }
+ return rowsAffected, nil
+}
+
+// QueryConn queries a SQL statement in a given connection.
+func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string, queryContext db.QueryContext) ([]*v1pb.QueryResult, error) {
+ singleSQLs, err := base.SplitMultiSQL(d.dbType, statement)
+ if err != nil {
+ return nil, err
+ }
+ singleSQLs = base.FilterEmptyStatements(singleSQLs)
+ if len(singleSQLs) == 0 {
+ return nil, nil
+ }
+
+ connectionID, err := getConnectionID(ctx, conn)
+ if err != nil {
+ return nil, err
+ }
+ slog.Debug("connectionID", slog.String("connectionID", connectionID))
+
+ var results []*v1pb.QueryResult
+ for _, singleSQL := range singleSQLs {
+ statement := singleSQL.Text
+ if queryContext.Explain {
+ explained, err := base.ExplainStatement(d.dbType, statement, db.ExplainFormat(queryContext.Option.GetExplainFormat()))
+ if err != nil {
+ return nil, err
+ }
+ statement = explained
+ } else if queryContext.Limit > 0 {
+ statement = base.StatementWithResultLimit(d.dbType, statement, queryContext.Limit, "")
+ }
+ sqlWithBytebaseAppComment := util.MySQLPrependBytebaseAppComment(statement)
+
+ _, allQuery, err := base.ValidateSQLForEditor(d.dbType, statement)
+ if err != nil {
+ // TODO(d): need to make parser compatible.
+ slog.Error("failed to validate sql", slog.String("statement", statement), log.BBError(err))
+ allQuery = true
+ }
+ startTime := time.Now()
+ queryResult, err := func() (*v1pb.QueryResult, error) {
+ if allQuery {
+ rows, err := conn.QueryContext(ctx, sqlWithBytebaseAppComment)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+ r, err := util.RowsToQueryResult(rows, makeValueByTypeName, convertValue, queryContext.MaximumSQLResultSize)
+ if err != nil {
+ return nil, err
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return r, nil
+ }
+
+ sqlResult, err := conn.ExecContext(ctx, statement)
+ if err != nil {
+ return nil, err
+ }
+ affectedRows, err := sqlResult.RowsAffected()
+ if err != nil {
+ slog.Info("rowsAffected returns error", log.BBError(err))
+ }
+ return util.BuildAffectedRowsResult(affectedRows, nil), nil
+ }()
+ stop := false
+ if err != nil {
+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {
+ slog.Info("cancel connection", slog.String("connectionID", connectionID))
+ if err := d.StopConnectionByID(connectionID); err != nil {
+ slog.Error("failed to cancel connection", slog.String("connectionID", connectionID), log.BBError(err))
+ }
+ }
+ queryResult = &v1pb.QueryResult{
+ Error: err.Error(),
+ }
+ stop = true
+ }
+ queryResult.Statement = statement
+ queryResult.Latency = durationpb.New(time.Since(startTime))
+ queryResult.RowsCount = int64(len(queryResult.Rows))
+ results = append(results, queryResult)
+ if stop {
+ break
+ }
+ }
+
+ return results, nil
+}
+
+func (d *Driver) StopConnectionByID(id string) error {
+ _, err := d.db.Exec(fmt.Sprintf("KILL QUERY %s", id)) // NOSONAR(go:S2077) id is from CONNECTION_ID() server function, not user input
+ return err
+}
+
+func containsDelimiterDirective(s string) bool {
+ for _, line := range strings.Split(s, "\n") {
+ trimmed := strings.TrimLeft(line, " \t")
+ upper := strings.ToUpper(trimmed)
+ if strings.HasPrefix(upper, "DELIMITER ") || strings.HasPrefix(upper, "DELIMITER\t") {
+ return true
+ }
+ }
+ return false
+}
+
+func getConnectionID(ctx context.Context, conn *sql.Conn) (string, error) {
+ var id string
+ if err := conn.QueryRowContext(ctx, `SELECT CONNECTION_ID();`).Scan(&id); err != nil {
+ return "", err
+ }
+ return id, nil
+}
diff --git a/backend/plugin/db/doris/doris_test.go b/backend/plugin/db/doris/doris_test.go
new file mode 100644
index 0000000000..178db87cdd
--- /dev/null
+++ b/backend/plugin/db/doris/doris_test.go
@@ -0,0 +1,87 @@
+package doris
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+
+ // Register the Doris editor validator.
+ _ "github.com/bytebase/bytebase/backend/plugin/parser/doris"
+)
+
+func TestParseVersion(t *testing.T) {
+ tests := []struct {
+ version string
+ want string
+ wantRest string
+ }{
+ {
+ version: "8.0.27",
+ want: "8.0.27",
+ wantRest: "",
+ },
+ {
+ version: "5.7.22-log",
+ want: "5.7.22",
+ wantRest: "-log",
+ },
+ {
+ version: "5.6.29_ddm_3.0.1.7",
+ want: "5.6.29",
+ wantRest: "_ddm_3.0.1.7",
+ },
+ {
+ version: "10.4.7-MariaDB",
+ want: "10.4.7",
+ wantRest: "-MariaDB",
+ },
+ }
+
+ a := require.New(t)
+ for _, tc := range tests {
+ version, rest, err := parseVersion(tc.version)
+ a.NoError(err)
+ a.Equal(tc.want, version)
+ a.Equal(tc.wantRest, rest)
+ }
+}
+
+func TestContainsDelimiterDirective(t *testing.T) {
+ tests := []struct {
+ name string
+ sql string
+ want bool
+ }{
+ {"basic directive", "DELIMITER ;;\nCREATE PROCEDURE p() BEGIN SELECT 1; END;;\nDELIMITER ;", true},
+ {"indented directive", " DELIMITER //\nSELECT 1//\nDELIMITER ;", true},
+ {"tab-indented directive", "\tDELIMITER $$\n", true},
+ {"string literal not a directive", "INSERT INTO t VALUES ('DELIMITER ');\nSELECT 1;", false},
+ {"comment not a directive", "-- DELIMITER //\nSELECT 1;", false},
+ {"column name not a directive", "SELECT delimiter FROM t;", false},
+ {"no delimiter at all", "SELECT 1;\nSELECT 2;", false},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ got := containsDelimiterDirective(tt.sql)
+ require.Equal(t, tt.want, got)
+ })
+ }
+}
+
+// TestValidateSQLForEditorUsesDorisGrammar pins this driver to the Doris parser: a
+// StarRocks-only spelling must be refused rather than quietly accepted, which is what
+// routing the connection through a sibling engine's grammar would do.
+func TestValidateSQLForEditorUsesDorisGrammar(t *testing.T) {
+ // StarRocks generated column AS (expr); Doris wants GENERATED ALWAYS AS.
+ const starRocksOnlyDDL = "CREATE TABLE t (a INT, b INT AS (a + 1)) DUPLICATE KEY(a) DISTRIBUTED BY HASH(a)"
+ _, _, err := base.ValidateSQLForEditor(storepb.Engine_DORIS, starRocksOnlyDDL)
+ require.Error(t, err)
+
+ // A Doris CREATE TABLE parses, and is DDL rather than a read-only query.
+ _, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_DORIS, "CREATE TABLE t (a INT) DUPLICATE KEY(a) DISTRIBUTED BY HASH(a)")
+ require.NoError(t, err)
+ require.False(t, allQuery)
+}
diff --git a/backend/plugin/db/doris/dump.go b/backend/plugin/db/doris/dump.go
new file mode 100644
index 0000000000..7c70a54298
--- /dev/null
+++ b/backend/plugin/db/doris/dump.go
@@ -0,0 +1,387 @@
+package doris
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "io"
+ "log/slog"
+ "strings"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/go-sql-driver/mysql"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common"
+)
+
+const (
+ // MySQL error code for "Unknown table".
+ errUnknownTable = 1051
+)
+
+// Dump and restore.
+const (
+ tableStmtFmt = "" +
+ "--\n" +
+ "-- Table structure for `%s`\n" +
+ "--\n" +
+ "%s;\n"
+ viewStmtFmt = "" +
+ "--\n" +
+ "-- View structure for `%s`\n" +
+ "--\n" +
+ "%s;\n"
+ materializedViewStmtFmt = "" +
+ "--\n" +
+ "-- Materialized view structure for `%s`\n" +
+ "--\n" +
+ "%s;\n"
+ tempViewStmtFmt = "" +
+ "--\n" +
+ "-- Temporary view structure for `%s`\n" +
+ "--\n" +
+ "%s\n"
+ tempMaterializedViewStmtFmt = "" +
+ "--\n" +
+ "-- Temporary view structure for materialized view `%s`\n" +
+ "--\n" +
+ "%s\n"
+ disableUniqueAndForeignKeyCheckStmt = "SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;\nSET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;\n"
+ restoreUniqueAndForeignKeyCheckStmt = "SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;\nSET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;\n"
+)
+
+// Dump dumps the database.
+func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {
+ slog.Debug("begin to dump database", slog.String("database", d.databaseName))
+
+ db := d.GetDB()
+ database := d.databaseName
+
+ // Disable foreign key check.
+ // mysqldump uses the same mechanism. When there is any schema or data dependency, we have to disable
+ // the unique and foreign key check so that the restoring will not fail.
+ if _, err := io.WriteString(out, disableUniqueAndForeignKeyCheckStmt); err != nil {
+ return err
+ }
+
+ // Table and view statement.
+ // We have to dump the table before views because of the structure dependency.
+ tables, err := getTables(db, database)
+ if err != nil {
+ return errors.Wrapf(err, "failed to get tables of database %q", database)
+ }
+ // Construct temporal views and materialized views.
+ // Create a temporary view with the same name as the view and with columns of
+ // the same name in order to satisfy views that depend on this view.
+ // This temporary view will be removed when the actual view is created.
+ // The properties of each column, are not preserved in this temporary
+ // view. They are not necessary because other views only need to reference
+ // the column name, thus we generate SELECT 1 AS colName1, 1 AS colName2.
+ // This will not be necessary once we can determine dependencies
+ // between views and can simply dump them in the appropriate order.
+ // https://sourcegraph.com/github.com/mysql/mysql-server/-/blob/client/mysqldump.cc?L2781
+ for _, tbl := range tables {
+ if tbl.TableType != viewTableType && tbl.TableType != materializedViewType {
+ continue
+ }
+ if tbl.InvalidView != "" {
+ // We will write the invalid view error string to schema.
+ stmtFmt := viewStmtFmt
+ if tbl.TableType == materializedViewType {
+ stmtFmt = materializedViewStmtFmt
+ }
+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", fmt.Sprintf(stmtFmt, tbl.Name, fmt.Sprintf("-- %s", tbl.InvalidView)))); err != nil {
+ return err
+ }
+ } else {
+ tempView := getTemporaryView(tbl.Name, tbl.ViewColumns)
+ if tbl.TableType == materializedViewType {
+ tempView = getTemporaryMaterializedView(tbl.Name, tbl.ViewColumns)
+ }
+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tempView)); err != nil {
+ return err
+ }
+ }
+ }
+ // Construct tables.
+ for _, tbl := range tables {
+ if tbl.TableType == viewTableType || tbl.TableType == materializedViewType {
+ continue
+ }
+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.Statement)); err != nil {
+ return err
+ }
+ }
+ // Construct final views, then materialized views (see finalEmitOrder: an MV is
+ // materialized from its sources at CREATE time, so its source views must be real
+ // first, not the temporary SELECT 1 placeholders).
+ for _, tbl := range finalEmitOrder(tables) {
+ // The temporary placeholder created above is always a regular view (even for
+ // materialized views — see getTemporaryMaterializedView), so it must be dropped
+ // with DROP VIEW. DROP MATERIALIZED VIEW would not match the placeholder, leaving
+ // it in the shared namespace so the real CREATE MATERIALIZED VIEW below collides.
+ if _, err := io.WriteString(out, dropPlaceholderStmt(tbl.Name)); err != nil {
+ return err
+ }
+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.Statement)); err != nil {
+ return err
+ }
+ }
+
+ // Restore foreign key check.
+ if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// finalEmitOrder returns the views and materialized views in the order their real
+// definitions must be emitted: all regular views first, then materialized views, each
+// group keeping its original relative order. information_schema.tables is unordered, and
+// a materialized view can be defined on a regular view and is materialized from its
+// sources at CREATE time, so its source views must already be real (not the temporary
+// SELECT 1 placeholders). Base tables are emitted separately and are excluded here.
+//
+// MVs keep their incoming order relative to each other, so a materialized view built on
+// another materialized view (nested async MVs) is not yet dependency-ordered and could
+// restore inactive if listed before its parent — tracked in BYT-9718.
+func finalEmitOrder(tables []*TableSchema) []*TableSchema {
+ var views, materializedViews []*TableSchema
+ for _, tbl := range tables {
+ switch tbl.TableType {
+ case viewTableType:
+ views = append(views, tbl)
+ case materializedViewType:
+ materializedViews = append(materializedViews, tbl)
+ default:
+ // Base tables and any other types are emitted elsewhere; skip here.
+ }
+ }
+ return append(views, materializedViews...)
+}
+
+// dropPlaceholderStmt returns the statement that drops the temporary placeholder emitted
+// for a view or materialized view before its real definition. The placeholder is always a
+// regular view (see getTemporaryMaterializedView), so it is dropped with DROP VIEW
+// regardless of the final object type.
+func dropPlaceholderStmt(name string) string {
+ return fmt.Sprintf("DROP VIEW IF EXISTS `%s`;\n", name)
+}
+
+func getTemporaryView(name string, columns []string) string {
+ var parts []string
+ for _, col := range columns {
+ parts = append(parts, fmt.Sprintf("1 AS `%s`", col))
+ }
+ stmt := fmt.Sprintf("CREATE VIEW `%s` AS SELECT\n %s;\n", name, strings.Join(parts, ",\n "))
+ return fmt.Sprintf(tempViewStmtFmt, name, stmt)
+}
+
+func getTemporaryMaterializedView(name string, columns []string) string {
+ var parts []string
+ for _, col := range columns {
+ parts = append(parts, fmt.Sprintf("1 AS `%s`", col))
+ }
+ // For materialized views, we create a temporary regular view as a placeholder
+ // since the actual materialized view will be created later with the full definition.
+ stmt := fmt.Sprintf("CREATE VIEW `%s` AS SELECT\n %s;\n", name, strings.Join(parts, ",\n "))
+ return fmt.Sprintf(tempMaterializedViewStmtFmt, name, stmt)
+}
+
+// TableSchema describes the schema of a table or view.
+type TableSchema struct {
+ Name string
+ TableType string
+ Statement string
+ ViewColumns []string
+ // InvalidView is the error message indicating an invalid view object.
+ InvalidView string
+}
+
+// getTables gets all tables of a database using the provided database instance.
+func getTables(db *sql.DB, dbName string) ([]*TableSchema, error) {
+ var tables []*TableSchema
+ query := "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?"
+ rows, err := db.Query(query, dbName)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ columns, err := rows.Columns()
+ if err != nil {
+ return nil, err
+ }
+ for rows.Next() {
+ var tbl TableSchema
+ // Some versions add a third, unused storage-format column.
+ var unusedStorageFormat string
+ if len(columns) == 3 {
+ if err := rows.Scan(&tbl.Name, &tbl.TableType, &unusedStorageFormat); err != nil {
+ return nil, err
+ }
+ } else {
+ if err := rows.Scan(&tbl.Name, &tbl.TableType); err != nil {
+ return nil, err
+ }
+ }
+ tables = append(tables, &tbl)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+
+ // Materialized views need a separate catalog lookup and re-tagging so the dump emits
+ // SHOW CREATE MATERIALIZED VIEW for them, since Doris reports them as BASE TABLE.
+ materializedViewNames, err := getMaterializedViewNames(db, dbName)
+ if err != nil {
+ return nil, err
+ }
+ markMaterializedViews(tables, materializedViewNames)
+
+ var result []*TableSchema
+ for _, tbl := range tables {
+ stmt, err := getTableStmt(db, dbName, tbl.Name, tbl.TableType)
+ if err != nil {
+ // Skip tables that were dropped between listing and fetching CREATE statement.
+ var mysqlErr *mysql.MySQLError
+ if errors.As(err, &mysqlErr) && mysqlErr.Number == errUnknownTable {
+ slog.Debug("table was dropped during dump, skipping",
+ slog.String("database", dbName),
+ slog.String("table", tbl.Name))
+ continue
+ }
+ return nil, errors.Wrapf(err, "failed to call getTableStmt(%q, %q, %q)", dbName, tbl.Name, tbl.TableType)
+ }
+ tbl.Statement = stmt
+ if tbl.TableType == viewTableType || tbl.TableType == materializedViewType {
+ viewColumns, err := getViewColumns(db, dbName, tbl.Name)
+ if err != nil {
+ tbl.InvalidView = err.Error()
+ } else {
+ tbl.ViewColumns = viewColumns
+ }
+ }
+ result = append(result, tbl)
+ }
+ return result, nil
+}
+
+// getMaterializedViewNames returns the set of materialized-view names in the database.
+// The query is best-effort: an unsupported version logs and yields an empty set.
+func getMaterializedViewNames(db *sql.DB, dbName string) (map[string]bool, error) {
+ names := make(map[string]bool)
+
+ query := fmt.Sprintf(`SELECT Name FROM mv_infos("database"="%s")`, dbName)
+
+ rows, err := db.Query(query)
+ if err != nil {
+ // The catalog may be unavailable on older engine versions; log and continue.
+ slog.Debug("failed to query materialized views, might not be supported", slog.String("error", err.Error()))
+ return names, nil
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ var name string
+ if err := rows.Scan(&name); err != nil {
+ return nil, err
+ }
+ names[name] = true
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return names, nil
+}
+
+// markMaterializedViews re-tags rows that are actually materialized views so the dump
+// emits SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE', which
+// becomes 'MATERIALIZED VIEW' when the name is in the materialized-view set. Regular
+// tables and views are left untouched.
+func markMaterializedViews(tables []*TableSchema, materializedViewNames map[string]bool) {
+ for _, tbl := range tables {
+ if !materializedViewNames[tbl.Name] {
+ continue
+ }
+ if tbl.TableType == baseTableType || tbl.TableType == viewTableType {
+ tbl.TableType = materializedViewType
+ }
+ }
+}
+
+// getTableStmt gets the create statement of a table.
+func getTableStmt(db *sql.DB, dbName, tblName, tblType string) (string, error) {
+ switch tblType {
+ case baseTableType:
+ query := fmt.Sprintf("SHOW CREATE TABLE `%s`.`%s`;", dbName, tblName)
+ var stmt, unused string
+ if err := db.QueryRow(query).Scan(&unused, &stmt); err != nil {
+ if err == sql.ErrNoRows {
+ return "", common.FormatDBErrorEmptyRowWithQuery(query)
+ }
+ return "", err
+ }
+ // Remove trailing semicolon if present, as the format string will add one
+ stmt = strings.TrimSuffix(stmt, ";")
+ return fmt.Sprintf(tableStmtFmt, tblName, stmt), nil
+ case viewTableType:
+ // This differs from mysqldump as it includes.
+ query := fmt.Sprintf("SHOW CREATE VIEW `%s`.`%s`;", dbName, tblName)
+ var createStmt, unused string
+ if err := db.QueryRow(query).Scan(&unused, &createStmt, &unused, &unused); err != nil {
+ if err == sql.ErrNoRows {
+ return "", common.FormatDBErrorEmptyRowWithQuery(query)
+ }
+ return "", err
+ }
+ // Remove trailing semicolon if present, as the format string will add one
+ createStmt = strings.TrimSuffix(createStmt, ";")
+ return fmt.Sprintf(viewStmtFmt, tblName, createStmt), nil
+ case materializedViewType:
+ // For materialized views we use SHOW CREATE MATERIALIZED VIEW.
+ // This command returns 2 columns: materialized view name and create statement.
+ query := fmt.Sprintf("SHOW CREATE MATERIALIZED VIEW `%s`.`%s`;", dbName, tblName)
+ var createStmt, unused string
+ if err := db.QueryRow(query).Scan(&unused, &createStmt); err != nil {
+ if err == sql.ErrNoRows {
+ return "", common.FormatDBErrorEmptyRowWithQuery(query)
+ }
+ return "", err
+ }
+ // Remove trailing semicolon if present, as the format string will add one
+ createStmt = strings.TrimSuffix(createStmt, ";")
+ return fmt.Sprintf(materializedViewStmtFmt, tblName, createStmt), nil
+ default:
+ slog.Debug("skip unsupported table type", slog.String("dbName", dbName), slog.String("tableName", tblName), slog.String("tableType", tblType))
+ return "", nil
+ }
+}
+
+// getViewColumns gets the create statement of a table.
+func getViewColumns(db *sql.DB, dbName, tblName string) ([]string, error) {
+ query := fmt.Sprintf("SHOW COLUMNS FROM `%s`.`%s`;", dbName, tblName)
+ // https://dev.mysql.com/doc/refman/8.0/en/show-columns.html
+ // Field, Type, Null, Key, Default, Extra.
+ rows, err := db.Query(query)
+ if err != nil {
+ return nil, err
+ }
+ defer rows.Close()
+
+ var fields []string
+ for rows.Next() {
+ var field string
+ var unused sql.NullString
+ if err := rows.Scan(&field, &unused, &unused, &unused, &unused, &unused); err != nil {
+ return nil, err
+ }
+ fields = append(fields, field)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+ return fields, nil
+}
diff --git a/backend/plugin/db/doris/dump_routine_probe_test.go b/backend/plugin/db/doris/dump_routine_probe_test.go
new file mode 100644
index 0000000000..6443433ed3
--- /dev/null
+++ b/backend/plugin/db/doris/dump_routine_probe_test.go
@@ -0,0 +1,143 @@
+package doris
+
+import (
+ "bytes"
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "fmt"
+ "io"
+ "regexp"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/go-sql-driver/mysql"
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+// BYT-10211: Doris has no MySQL stored routines or events, and Doris
+// kernels that dropped the PL/SQL grammar (SelectDB 4.0.12+/4.1.8+/5.0.5+, apache/doris#58700)
+// reject `SHOW FUNCTION STATUS` outright, which aborted the whole schema sync. The dump must
+// complete without ever sending the MySQL routine or event probes.
+func TestDumpSendsNoRoutineOrEventProbe(t *testing.T) {
+ dumpRecorder.reset(plsqlStrippedCatalog())
+ sqlDB, err := sql.Open(dumpRecorderDriverName, "")
+ require.NoError(t, err)
+ defer sqlDB.Close()
+
+ d := &Driver{dbType: storepb.Engine_DORIS, db: sqlDB, databaseName: "analytics"}
+ var out bytes.Buffer
+ require.NoError(t, d.Dump(context.Background(), &out, nil))
+
+ require.Contains(t, out.String(), "CREATE TABLE `orders`")
+ require.Contains(t, out.String(), "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`")
+ for _, q := range dumpRecorder.recorded() {
+ require.NotRegexp(t, forbiddenProbe, q)
+ }
+}
+
+var (
+ routineStatusProbe = regexp.MustCompile(`(?i)^\s*SHOW\s+(FUNCTION|PROCEDURE)\s+STATUS`)
+ forbiddenProbe = regexp.MustCompile(`(?i)SHOW\s+((FUNCTION|PROCEDURE)\s+STATUS|EVENTS|CREATE\s+(FUNCTION|PROCEDURE|EVENT))`)
+)
+
+// plsqlStrippedCatalog answers the dump's table and view queries with the column shapes a
+// live Doris 4.1.3 returns, and rejects the routine probes with the exact error a
+// PL/SQL-stripped kernel returns.
+func plsqlStrippedCatalog() func(string) ([]string, [][]driver.Value, error) {
+ return func(query string) ([]string, [][]driver.Value, error) {
+ switch {
+ case routineStatusProbe.MatchString(query):
+ return nil, nil, &mysql.MySQLError{Number: 1105, SQLState: [5]byte{'H', 'Y', '0', '0', '0'}, Message: "errCode = 2, detailMessage = no viable alternative at input 'SHOW FUNCTION'(line 1, pos 5)"}
+ case strings.Contains(query, "information_schema.TABLES"):
+ return []string{"TABLE_NAME", "TABLE_TYPE"}, [][]driver.Value{{"orders", "BASE TABLE"}, {"v_orders", "VIEW"}}, nil
+ case strings.Contains(query, "mv_infos("):
+ return []string{"Name"}, nil, nil
+ case strings.HasPrefix(query, "SHOW CREATE TABLE"):
+ return []string{"Table", "Create Table"}, [][]driver.Value{{"orders", "CREATE TABLE `orders` (\n `id` int NULL\n) ENGINE=OLAP"}}, nil
+ case strings.HasPrefix(query, "SHOW CREATE VIEW"):
+ return []string{"View", "Create View", "character_set_client", "collation_connection"}, [][]driver.Value{{"v_orders", "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`", "utf8", "utf8_general_ci"}}, nil
+ case strings.HasPrefix(query, "SHOW COLUMNS FROM"):
+ return []string{"Field", "Type", "Null", "Key", "Default", "Extra"}, [][]driver.Value{{"id", "int", "YES", "", nil, ""}}, nil
+ }
+ return nil, nil, errors.Errorf("unexpected statement: %q", query)
+ }
+}
+
+// dumpRecorder is registered once: database/sql panics on a repeated driver name, which
+// would take the whole package down under `go test -count=2`. Each subtest resets it.
+const dumpRecorderDriverName = "doris-dump-recorder"
+
+var dumpRecorder recordingDriver
+
+func init() {
+ sql.Register(dumpRecorderDriverName, &dumpRecorder)
+}
+
+// recordingDriver is a database/sql driver that records every statement and answers it
+// from a canned function, so Dump runs against a scripted server without a network.
+type recordingDriver struct {
+ mu sync.Mutex
+ queries []string
+ answer func(query string) ([]string, [][]driver.Value, error)
+}
+
+func (d *recordingDriver) Open(string) (driver.Conn, error) { return &recordingConn{d: d}, nil }
+
+func (d *recordingDriver) reset(answer func(string) ([]string, [][]driver.Value, error)) {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ d.queries = nil
+ d.answer = answer
+}
+
+func (d *recordingDriver) recorded() []string {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ return append([]string(nil), d.queries...)
+}
+
+type recordingConn struct{ d *recordingDriver }
+
+func (*recordingConn) Prepare(string) (driver.Stmt, error) {
+ return nil, errors.New("prepare is not supported")
+}
+func (*recordingConn) Close() error { return nil }
+func (*recordingConn) Begin() (driver.Tx, error) { return nil, errors.New("tx is not supported") }
+
+// QueryContext makes database/sql skip Prepare for both db.Query(q) and db.Query(q, arg);
+// the driver interpolates the argument itself, as the real DSN (interpolateParams=true) does.
+func (c *recordingConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
+ for _, a := range args {
+ query = strings.Replace(query, "?", fmt.Sprintf("'%v'", a.Value), 1)
+ }
+ c.d.mu.Lock()
+ c.d.queries = append(c.d.queries, query)
+ c.d.mu.Unlock()
+ cols, rows, err := c.d.answer(query)
+ if err != nil {
+ return nil, err
+ }
+ return &sliceRows{cols: cols, rows: rows}, nil
+}
+
+type sliceRows struct {
+ cols []string
+ rows [][]driver.Value
+ i int
+}
+
+func (r *sliceRows) Columns() []string { return r.cols }
+func (*sliceRows) Close() error { return nil }
+func (r *sliceRows) Next(dest []driver.Value) error {
+ if r.i >= len(r.rows) {
+ return io.EOF
+ }
+ copy(dest, r.rows[r.i])
+ r.i++
+ return nil
+}
diff --git a/backend/plugin/db/doris/dump_test.go b/backend/plugin/db/doris/dump_test.go
new file mode 100644
index 0000000000..0739b855df
--- /dev/null
+++ b/backend/plugin/db/doris/dump_test.go
@@ -0,0 +1,67 @@
+package doris
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+)
+
+func TestGetTemporaryView(t *testing.T) {
+ a := require.New(t)
+ got := getTemporaryView("db1", []string{"col1", "col2"})
+ want := "--\n-- Temporary view structure for `db1`\n--\nCREATE VIEW `db1` AS SELECT\n 1 AS `col1`,\n 1 AS `col2`;\n\n"
+ a.Equal(want, got)
+}
+
+// BYT-9689: the dump path re-tags materialized-view rows so it emits
+// SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE', and some
+// setups as 'VIEW'; either must become 'MATERIALIZED VIEW' when the name is a known MV,
+// while regular tables and views are left untouched.
+func TestMarkMaterializedViews(t *testing.T) {
+ tables := []*TableSchema{
+ {Name: "mv_base", TableType: baseTableType}, // MV reported as BASE TABLE
+ {Name: "mv_view", TableType: viewTableType}, // MV reported as VIEW
+ {Name: "v_regular", TableType: viewTableType},
+ {Name: "t_plain", TableType: baseTableType},
+ }
+ markMaterializedViews(tables, map[string]bool{"mv_base": true, "mv_view": true})
+
+ require.Equal(t, materializedViewType, tables[0].TableType)
+ require.Equal(t, materializedViewType, tables[1].TableType)
+ require.Equal(t, viewTableType, tables[2].TableType, "regular view must stay a view")
+ require.Equal(t, baseTableType, tables[3].TableType, "plain table must stay a table")
+}
+
+// BYT-9689: the dump emits a temporary regular-view placeholder for every view AND
+// materialized view (getTemporaryMaterializedView creates a CREATE VIEW), then drops it
+// before emitting the real definition. The drop must therefore be DROP VIEW even for an
+// MV — DROP MATERIALIZED VIEW would not match the placeholder, leaving it in the shared
+// namespace so the real CREATE MATERIALIZED VIEW collides with it on replay.
+func TestMaterializedViewPlaceholderIsDroppedAsView(t *testing.T) {
+ placeholder := getTemporaryMaterializedView("mv1", []string{"a"})
+ require.Contains(t, placeholder, "CREATE VIEW `mv1`")
+ require.NotContains(t, placeholder, "CREATE MATERIALIZED VIEW")
+
+ require.Equal(t, "DROP VIEW IF EXISTS `mv1`;\n", dropPlaceholderStmt("mv1"))
+}
+
+// BYT-9689: information_schema.tables is unordered, so an MV can be listed before a view
+// it is defined on. A materialized view is materialized from its sources at CREATE time
+// (unlike a lazy regular view), so all regular views must be emitted — and on restore,
+// created real — before any materialized view; otherwise the MV would materialize from
+// the temporary SELECT 1 placeholder. finalEmitOrder partitions views ahead of MVs while
+// preserving each group's relative order, and drops base tables (emitted elsewhere).
+func TestFinalEmitOrder_ViewsBeforeMaterializedViews(t *testing.T) {
+ tables := []*TableSchema{
+ {Name: "mv1", TableType: materializedViewType},
+ {Name: "v1", TableType: viewTableType},
+ {Name: "t1", TableType: baseTableType},
+ {Name: "mv2", TableType: materializedViewType},
+ {Name: "v2", TableType: viewTableType},
+ }
+ var names []string
+ for _, tbl := range finalEmitOrder(tables) {
+ names = append(names, tbl.Name)
+ }
+ require.Equal(t, []string{"v1", "v2", "mv1", "mv2"}, names)
+}
diff --git a/backend/plugin/db/doris/query.go b/backend/plugin/db/doris/query.go
new file mode 100644
index 0000000000..4e5150015a
--- /dev/null
+++ b/backend/plugin/db/doris/query.go
@@ -0,0 +1,76 @@
+package doris
+
+import (
+ "database/sql"
+
+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"
+ "github.com/bytebase/bytebase/backend/plugin/db/util"
+)
+
+func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {
+ switch typeName {
+ case "VARCHAR", "TEXT", "UUID", "DATETIME", "TIMESTAMP":
+ return new(sql.NullString)
+ case "BOOL":
+ return new(sql.NullBool)
+ case "INT", "INTEGER", "TINYINT", "SMALLINT", "MEDIUMINT", "BIGINT", "INT2", "INT4", "INT8":
+ return new(sql.NullInt64)
+ case "FLOAT", "DOUBLE", "FLOAT4", "FLOAT8":
+ return new(sql.NullFloat64)
+ case "BIT", "VARBIT":
+ return new([]byte)
+ default:
+ return new(sql.NullString)
+ }
+}
+
+func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.RowValue {
+ switch raw := value.(type) {
+ case *sql.NullString:
+ if raw.Valid {
+ if typeName == "TIMESTAMP" || typeName == "DATETIME" {
+ _, scale, _ := columnType.DecimalSize()
+ return util.BuildTimestampOrStringRowValue(raw.String, scale)
+ }
+ return &v1pb.RowValue{
+ Kind: &v1pb.RowValue_StringValue{
+ StringValue: raw.String,
+ },
+ }
+ }
+ case *sql.NullInt64:
+ if raw.Valid {
+ return &v1pb.RowValue{
+ Kind: &v1pb.RowValue_Int64Value{
+ Int64Value: raw.Int64,
+ },
+ }
+ }
+ case *[]byte:
+ if len(*raw) > 0 {
+ return &v1pb.RowValue{
+ Kind: &v1pb.RowValue_BytesValue{
+ BytesValue: *raw,
+ },
+ }
+ }
+ case *sql.NullBool:
+ if raw.Valid {
+ return &v1pb.RowValue{
+ Kind: &v1pb.RowValue_BoolValue{
+ BoolValue: raw.Bool,
+ },
+ }
+ }
+ case *sql.NullFloat64:
+ if raw.Valid {
+ return &v1pb.RowValue{
+ Kind: &v1pb.RowValue_DoubleValue{
+ DoubleValue: raw.Float64,
+ },
+ }
+ }
+ default:
+ }
+ return util.NullRowValue
+}
diff --git a/backend/plugin/db/doris/sync.go b/backend/plugin/db/doris/sync.go
new file mode 100644
index 0000000000..001e309248
--- /dev/null
+++ b/backend/plugin/db/doris/sync.go
@@ -0,0 +1,387 @@
+package doris
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "log/slog"
+ "strconv"
+ "strings"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/pkg/errors"
+
+ "github.com/bytebase/bytebase/backend/common"
+ "github.com/bytebase/bytebase/backend/common/log"
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/db"
+ "github.com/bytebase/bytebase/backend/plugin/db/mysql"
+ "github.com/bytebase/bytebase/backend/plugin/db/util"
+)
+
+const (
+ autoIncrementSymbol = "AUTO_INCREMENT"
+)
+
+// systemDatabaseExclusion returns the quoted, comma-joined system databases to
+// exclude from instance discovery, matching the parser-side system-database
+// classification. Doris has no `sys` metadatabase.
+func systemDatabaseExclusion() string {
+ return strings.Join([]string{"'information_schema'", "'_statistics_'"}, ", ")
+}
+
+// SyncInstance syncs the instance.
+func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error) {
+ version, _, err := d.getVersion(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ var lowerCaseTableNames int32
+ lowerCaseTableNamesText, err := d.getServerVariable(ctx, "lower_case_table_names")
+ if err != nil {
+ slog.Debug("failed to get lower_case_table_names variable", log.BBError(err))
+ } else {
+ v, err := strconv.ParseInt(lowerCaseTableNamesText, 10, 32)
+ if err != nil {
+ slog.Debug("failed to parse lower_case_table_names variable", log.BBError(err))
+ } else {
+ lowerCaseTableNames = int32(v)
+ }
+ }
+
+ // Query db info
+ where := fmt.Sprintf("LOWER(SCHEMA_NAME) NOT IN (%s)", systemDatabaseExclusion())
+ query := `
+ SELECT
+ SCHEMA_NAME,
+ DEFAULT_CHARACTER_SET_NAME,
+ DEFAULT_COLLATION_NAME
+ FROM information_schema.SCHEMATA
+ WHERE ` + where
+ rows, err := d.db.QueryContext(ctx, query)
+ if err != nil {
+ return nil, util.FormatErrorWithQuery(err, query)
+ }
+ defer rows.Close()
+
+ var databases []*metadatapb.DatabaseSchemaMetadata
+ for rows.Next() {
+ database := &metadatapb.DatabaseSchemaMetadata{}
+ if err := rows.Scan(
+ &database.Name,
+ &database.CharacterSet,
+ &database.Collation,
+ ); err != nil {
+ return nil, err
+ }
+ database.CharacterSet = ""
+ database.Collation = ""
+ databases = append(databases, database)
+ }
+ if err := rows.Err(); err != nil {
+ return nil, err
+ }
+
+ return &db.InstanceMetadata{
+ Version: version,
+ Databases: databases,
+ Metadata: &storepb.Instance{
+ MysqlLowerCaseTableNames: lowerCaseTableNames,
+ },
+ }, nil
+}
+
+func (d *Driver) getServerVariable(ctx context.Context, varName string) (string, error) {
+ db := d.GetDB()
+ query := fmt.Sprintf("SHOW VARIABLES LIKE '%s'", varName)
+ var varNameFound, value string
+ if err := db.QueryRowContext(ctx, query).Scan(&varNameFound, &value); err != nil {
+ if err == sql.ErrNoRows {
+ return "", common.FormatDBErrorEmptyRowWithQuery(query)
+ }
+ return "", util.FormatErrorWithQuery(err, query)
+ }
+ if varName != varNameFound {
+ return "", errors.Errorf("expecting variable %s, but got %s", varName, varNameFound)
+ }
+ return value, nil
+}
+
+// isMaterializedView reports whether a table row is a materialized view, based on its
+// table type and membership in the materialized-view set. Doris reports materialized
+// views with TABLE_TYPE='BASE TABLE', and some setups with 'MATERIALIZED VIEW'; in
+// either case, presence in materializedViewMap is the authoritative signal.
+func isMaterializedView(tableType string, key db.TableKey, materializedViewMap map[db.TableKey]*metadatapb.MaterializedViewMetadata) bool {
+ if _, ok := materializedViewMap[key]; !ok {
+ return false
+ }
+ switch tableType {
+ case baseTableType, viewTableType, materializedViewType:
+ return true
+ default:
+ return false
+ }
+}
+
+// getMaterializedViews returns the materialized views of the current database keyed by
+// table name. The query is best-effort: a version that does not support it is logged
+// and yields an empty set rather than failing the sync.
+func (d *Driver) getMaterializedViews(ctx context.Context) (map[db.TableKey]*metadatapb.MaterializedViewMetadata, error) {
+ materializedViewMap := make(map[db.TableKey]*metadatapb.MaterializedViewMetadata)
+
+ // Doris exposes materialized views via the mv_infos() table-valued function.
+ // https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/mv_infos
+ query := fmt.Sprintf(`SELECT Name, QuerySql FROM mv_infos("database"="%s")`, d.databaseName)
+
+ rows, err := d.db.QueryContext(ctx, query)
+ if err != nil {
+ // The catalog may be unavailable on older engine versions; log and continue.
+ slog.Debug("failed to query materialized views, might not be supported in this version", log.BBError(err))
+ return materializedViewMap, nil
+ }
+ defer rows.Close()
+
+ for rows.Next() {
+ materializedView := &metadatapb.MaterializedViewMetadata{}
+ if err := rows.Scan(&materializedView.Name, &materializedView.Definition); err != nil {
+ return nil, err
+ }
+ key := db.TableKey{Schema: "", Table: materializedView.Name}
+ materializedViewMap[key] = materializedView
+ }
+ if err := rows.Err(); err != nil {
+ return nil, util.FormatErrorWithQuery(err, query)
+ }
+ return materializedViewMap, nil
+}
+
+// SyncDBSchema syncs a single database schema.
+func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMetadata, error) {
+ schemaMetadata := &metadatapb.SchemaMetadata{
+ Name: "",
+ }
+
+ // There is not yet a way to list indexes from information_schema.
+ // Query column info.
+ columnMap := make(map[db.TableKey][]*metadatapb.ColumnMetadata)
+ columnQuery := fmt.Sprintf(`
+ SELECT
+ TABLE_NAME,
+ IFNULL(COLUMN_NAME, ''),
+ ORDINAL_POSITION,
+ COLUMN_DEFAULT,
+ IS_NULLABLE,
+ COLUMN_TYPE,
+ IFNULL(CHARACTER_SET_NAME, ''),
+ IFNULL(COLLATION_NAME, ''),
+ COLUMN_COMMENT,
+ EXTRA
+ FROM information_schema.columns
+ WHERE TABLE_SCHEMA = '%s'
+ ORDER BY TABLE_NAME, ORDINAL_POSITION`, d.databaseName)
+ columnRows, err := d.db.QueryContext(ctx, columnQuery)
+ if err != nil {
+ return nil, util.FormatErrorWithQuery(err, columnQuery)
+ }
+ defer columnRows.Close()
+ for columnRows.Next() {
+ column := &metadatapb.ColumnMetadata{}
+ var tableName, nullable, extra string
+ var defaultStr sql.NullString
+ if err := columnRows.Scan(
+ &tableName,
+ &column.Name,
+ &column.Position,
+ &defaultStr,
+ &nullable,
+ &column.Type,
+ &column.CharacterSet,
+ &column.Collation,
+ &column.Comment,
+ &extra,
+ ); err != nil {
+ return nil, err
+ }
+ isNullBool, err := util.ConvertYesNo(nullable)
+ if err != nil {
+ return nil, err
+ }
+ if defaultStr.Valid {
+ // Doris is MySQL-compatible, so use MySQL's default handling approach
+ unquotedDefault := mysql.UnquoteMySQLString(defaultStr.String)
+ switch {
+ case mysql.IsCurrentTimestampLike(unquotedDefault):
+ column.Default = unquotedDefault
+ case strings.Contains(extra, "DEFAULT_GENERATED"):
+ unescapedDefault := mysql.UnescapeExpressionDefault(unquotedDefault)
+ column.Default = fmt.Sprintf("(%s)", unescapedDefault)
+ default:
+ // For non-generated and non CURRENT_XXX default value, preserve quotes for mysqldump compatibility
+ column.Default = defaultStr.String
+ }
+ } else if strings.Contains(strings.ToUpper(extra), autoIncrementSymbol) {
+ // TODO(zp): refactor column default value.
+ // Use the upper case to consistent with MySQL Dump.
+ column.Default = autoIncrementSymbol
+ } else if isNullBool {
+ // This is NULL if the column has an explicit default of NULL,
+ // or if the column definition includes no DEFAULT clause.
+ // https://dev.mysql.com/doc/refman/8.0/en/information-schema-columns-table.html
+ column.Default = "NULL"
+ }
+ column.Nullable = isNullBool
+
+ key := db.TableKey{Schema: "", Table: tableName}
+ columnMap[key] = append(columnMap[key], column)
+ }
+ if err := columnRows.Err(); err != nil {
+ return nil, util.FormatErrorWithQuery(err, columnQuery)
+ }
+
+ // Query view info.
+ viewMap := make(map[db.TableKey]*metadatapb.ViewMetadata)
+ viewQuery := fmt.Sprintf(`
+ SELECT
+ TABLE_NAME,
+ VIEW_DEFINITION
+ FROM information_schema.VIEWS
+ WHERE TABLE_SCHEMA = '%s'`, d.databaseName)
+ viewRows, err := d.db.QueryContext(ctx, viewQuery)
+ if err != nil {
+ return nil, util.FormatErrorWithQuery(err, viewQuery)
+ }
+ defer viewRows.Close()
+ for viewRows.Next() {
+ view := &metadatapb.ViewMetadata{}
+ if err := viewRows.Scan(
+ &view.Name,
+ &view.Definition,
+ ); err != nil {
+ return nil, err
+ }
+ key := db.TableKey{Schema: "", Table: view.Name}
+ view.Columns = columnMap[key]
+ viewMap[key] = view
+ }
+ if err := viewRows.Err(); err != nil {
+ return nil, util.FormatErrorWithQuery(err, viewQuery)
+ }
+
+ // Query materialized view info (engine-specific catalog; best-effort).
+ materializedViewMap, err := d.getMaterializedViews(ctx)
+ if err != nil {
+ return nil, err
+ }
+
+ // Query table info.
+ tableQuery := fmt.Sprintf(`
+ SELECT
+ TABLE_NAME,
+ TABLE_TYPE,
+ IFNULL(ENGINE, ''),
+ IFNULL(TABLE_COLLATION, ''),
+ IFNULL(TABLE_ROWS, 0),
+ IFNULL(DATA_LENGTH, 0),
+ IFNULL(INDEX_LENGTH, 0),
+ IFNULL(DATA_FREE, 0),
+ IFNULL(CREATE_OPTIONS, ''),
+ IFNULL(TABLE_COMMENT, '')
+ FROM information_schema.TABLES
+ WHERE TABLE_SCHEMA = '%s'
+ ORDER BY TABLE_NAME`, d.databaseName)
+ tableRows, err := d.db.QueryContext(ctx, tableQuery)
+ if err != nil {
+ return nil, util.FormatErrorWithQuery(err, tableQuery)
+ }
+ defer tableRows.Close()
+ for tableRows.Next() {
+ var tableName, tableType, engine, collation, createOptions, comment string
+ var rowCount, dataSize, indexSize, dataFree int64
+ // Workaround TiDB bug https://github.com/pingcap/tidb/issues/27970
+ var tableCollation sql.NullString
+ if err := tableRows.Scan(
+ &tableName,
+ &tableType,
+ &engine,
+ &collation,
+ &rowCount,
+ &dataSize,
+ &indexSize,
+ &dataFree,
+ &createOptions,
+ &comment,
+ ); err != nil {
+ return nil, err
+ }
+
+ key := db.TableKey{Schema: "", Table: tableName}
+ // Doris reports materialized views as TABLE_TYPE='BASE TABLE';
+ // route any row whose name is in the materialized-view set to MaterializedViews.
+ if isMaterializedView(tableType, key, materializedViewMap) {
+ materializedView := materializedViewMap[key]
+ materializedView.Comment = comment
+ schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, materializedView)
+ continue
+ }
+ switch tableType {
+ case baseTableType:
+ tableMetadata := &metadatapb.TableMetadata{
+ Name: tableName,
+ Columns: columnMap[key],
+ Engine: engine,
+ Collation: collation,
+ RowCount: rowCount,
+ DataSize: dataSize,
+ IndexSize: indexSize,
+ DataFree: dataFree,
+ CreateOptions: createOptions,
+ Comment: comment,
+ }
+ if tableCollation.Valid {
+ tableMetadata.Collation = tableCollation.String
+ }
+ // TODO(d): add index information whenever it is available.
+ schemaMetadata.Tables = append(schemaMetadata.Tables, tableMetadata)
+ case viewTableType:
+ if view, ok := viewMap[key]; ok {
+ view.Comment = comment
+ schemaMetadata.Views = append(schemaMetadata.Views, view)
+ }
+ default:
+ // Includes 'MATERIALIZED VIEW' rows absent from the materialized-view set
+ // and any other unrecognized type.
+ slog.Debug("skipping unhandled table type", slog.String("tableName", tableName), slog.String("tableType", tableType))
+ }
+ }
+ if err := tableRows.Err(); err != nil {
+ return nil, util.FormatErrorWithQuery(err, tableQuery)
+ }
+
+ databaseMetadata := &metadatapb.DatabaseSchemaMetadata{
+ Name: d.databaseName,
+ Schemas: []*metadatapb.SchemaMetadata{schemaMetadata},
+ }
+ // Query db info.
+ databaseQuery := fmt.Sprintf(`
+ SELECT
+ DEFAULT_CHARACTER_SET_NAME,
+ DEFAULT_COLLATION_NAME
+ FROM information_schema.SCHEMATA
+ WHERE SCHEMA_NAME = '%s'`, d.databaseName)
+ if err := d.db.QueryRowContext(ctx, databaseQuery).Scan(
+ &databaseMetadata.CharacterSet,
+ &databaseMetadata.Collation,
+ ); err != nil {
+ if err == sql.ErrNoRows {
+ return nil, common.Errorf(common.NotFound, "database %q not found", d.databaseName)
+ }
+ return nil, err
+ }
+ // "characterSet":"utf8\u0000", "collation":"utf8_general_ci\u0000".
+ // ERROR: unsupported Unicode escape sequence (SQLSTATE 22P05).
+ databaseMetadata.CharacterSet = ""
+ databaseMetadata.Collation = ""
+
+ return databaseMetadata, err
+}
diff --git a/backend/plugin/db/doris/sync_test.go b/backend/plugin/db/doris/sync_test.go
new file mode 100644
index 0000000000..439fef85de
--- /dev/null
+++ b/backend/plugin/db/doris/sync_test.go
@@ -0,0 +1,39 @@
+package doris
+
+import (
+ "testing"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+ "github.com/stretchr/testify/require"
+
+ "github.com/bytebase/bytebase/backend/plugin/db"
+)
+
+// BYT-9689: Doris reports materialized views in information_schema.tables with
+// TABLE_TYPE='BASE TABLE', and some setups with 'MATERIALIZED VIEW', so the table type
+// alone cannot identify one. Membership in the materialized-view set is the
+// authoritative signal, whatever type the row carries.
+func TestIsMaterializedView(t *testing.T) {
+ mvMap := map[db.TableKey]*metadatapb.MaterializedViewMetadata{
+ {Schema: "", Table: "mv_async"}: {Name: "mv_async"},
+ }
+ mvKey := db.TableKey{Schema: "", Table: "mv_async"}
+
+ require.True(t, isMaterializedView(baseTableType, mvKey, mvMap))
+ require.True(t, isMaterializedView(materializedViewType, mvKey, mvMap))
+ require.True(t, isMaterializedView(viewTableType, mvKey, mvMap))
+
+ // A name absent from the MV set is never a materialized view.
+ require.False(t, isMaterializedView(viewTableType, db.TableKey{Table: "v_regular"}, mvMap))
+ require.False(t, isMaterializedView(baseTableType, db.TableKey{Table: "sales"}, mvMap))
+}
+
+// #20566 review (P2): SyncInstance's system-database exclusion must match the parser's
+// system set. Doris has no `sys` metadatabase, so excluding it would risk hiding a user
+// database named sys.
+func TestSystemDatabaseExclusion(t *testing.T) {
+ got := systemDatabaseExclusion()
+ require.Contains(t, got, "'information_schema'")
+ require.Contains(t, got, "'_statistics_'")
+ require.NotContains(t, got, "'sys'")
+}
diff --git a/backend/plugin/db/starrocks/dump.go b/backend/plugin/db/starrocks/dump.go
index 412404eb78..de8fa71570 100644
--- a/backend/plugin/db/starrocks/dump.go
+++ b/backend/plugin/db/starrocks/dump.go
@@ -13,7 +13,6 @@ import (
"github.com/pkg/errors"
"github.com/bytebase/bytebase/backend/common"
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
const (
@@ -48,8 +47,6 @@ const (
"-- Temporary view structure for materialized view `%s`\n" +
"--\n" +
"%s\n"
- disableUniqueAndForeignKeyCheckStmt = "SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;\nSET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;\n"
- restoreUniqueAndForeignKeyCheckStmt = "SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;\nSET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;\n"
)
// Dump dumps the database.
@@ -58,23 +55,14 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSc
db := d.GetDB()
database := d.databaseName
- dbType := d.dbType
- // Disable foreign key check.
- // mysqldump uses the same mechanism. When there is any schema or data dependency, we have to disable
- // the unique and foreign key check so that the restoring will not fail.
- // StarRocks has no MySQL UNIQUE_CHECKS/FOREIGN_KEY_CHECKS session variables (and does not
- // enforce foreign keys), so emitting the guard makes the dump fail to replay with
- // "Unknown system variable 'UNIQUE_CHECKS'".
- if d.dbType != storepb.Engine_STARROCKS {
- if _, err := io.WriteString(out, disableUniqueAndForeignKeyCheckStmt); err != nil {
- return err
- }
- }
+ // mysqldump brackets a dump with UNIQUE_CHECKS/FOREIGN_KEY_CHECKS guards. StarRocks has
+ // neither session variable and does not enforce foreign keys, so emitting them makes the
+ // dump fail to replay with "Unknown system variable 'UNIQUE_CHECKS'".
// Table and view statement.
// We have to dump the table before views because of the structure dependency.
- tables, err := getTables(db, database, dbType)
+ tables, err := getTables(db, database)
if err != nil {
return errors.Wrapf(err, "failed to get tables of database %q", database)
}
@@ -136,13 +124,6 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSc
}
}
- // Restore foreign key check (skipped for StarRocks, which lacks these variables).
- if d.dbType != storepb.Engine_STARROCKS {
- if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {
- return err
- }
- }
-
return nil
}
@@ -210,7 +191,7 @@ type TableSchema struct {
}
// getTables gets all tables of a database using the provided database instance.
-func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema, error) {
+func getTables(db *sql.DB, dbName string) ([]*TableSchema, error) {
var tables []*TableSchema
query := "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?"
rows, err := db.Query(query, dbName)
@@ -225,10 +206,9 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema
}
for rows.Next() {
var tbl TableSchema
- // StorageFormat is the third but unused column for Doris.
+ // Some versions add a third, unused storage-format column.
var unusedStorageFormat string
if len(columns) == 3 {
- // Doris.
if err := rows.Scan(&tbl.Name, &tbl.TableType, &unusedStorageFormat); err != nil {
return nil, err
}
@@ -244,8 +224,8 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema
}
// Materialized views need a separate catalog lookup and re-tagging so the dump emits
- // SHOW CREATE MATERIALIZED VIEW for them (Doris reports them as BASE TABLE, StarRocks as VIEW).
- materializedViewNames, err := getMaterializedViewNames(db, dbName, dbType)
+ // SHOW CREATE MATERIALIZED VIEW for them, since StarRocks reports them as VIEW.
+ materializedViewNames, err := getMaterializedViewNames(db, dbName)
if err != nil {
return nil, err
}
@@ -280,23 +260,14 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema
}
// getMaterializedViewNames returns the set of materialized-view names in the database.
-// Doris and StarRocks expose them through different catalogs; the query is best-effort
-// (an unsupported engine/version logs and yields an empty set). StarRocks sync rollups
-// (REFRESH_TYPE='ROLLUP') are excluded — they are rollup indexes on the base table, not
-// standalone views.
-func getMaterializedViewNames(db *sql.DB, dbName string, dbType storepb.Engine) (map[string]bool, error) {
+// The query is best-effort: an unsupported version logs and yields an empty set. Sync
+// rollups (REFRESH_TYPE='ROLLUP') are excluded: they are rollup indexes on the base
+// table, not standalone views.
+func getMaterializedViewNames(db *sql.DB, dbName string) (map[string]bool, error) {
names := make(map[string]bool)
- var query string
- switch dbType {
- case storepb.Engine_DORIS:
- query = fmt.Sprintf(`SELECT Name FROM mv_infos("database"="%s")`, dbName)
- case storepb.Engine_STARROCKS:
- // IFNULL guards the bare-string scan below: database/sql errors on a NULL->string scan.
- query = fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, dbName)
- default:
- return names, nil
- }
+ // IFNULL guards the bare-string scan below: database/sql errors on a NULL->string scan.
+ query := fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, dbName)
rows, err := db.Query(query)
if err != nil {
@@ -307,18 +278,13 @@ func getMaterializedViewNames(db *sql.DB, dbName string, dbType storepb.Engine)
defer rows.Close()
for rows.Next() {
- var name string
- if dbType == storepb.Engine_STARROCKS {
- var refreshType string
- if err := rows.Scan(&name, &refreshType); err != nil {
- return nil, err
- }
- if isSyncRollup(refreshType) {
- continue
- }
- } else if err := rows.Scan(&name); err != nil {
+ var name, refreshType string
+ if err := rows.Scan(&name, &refreshType); err != nil {
return nil, err
}
+ if isSyncRollup(refreshType) {
+ continue
+ }
names[name] = true
}
if err := rows.Err(); err != nil {
@@ -328,9 +294,9 @@ func getMaterializedViewNames(db *sql.DB, dbName string, dbType storepb.Engine)
}
// markMaterializedViews re-tags rows that are actually materialized views so the dump
-// emits SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE' and
-// StarRocks as 'VIEW'; either becomes 'MATERIALIZED VIEW' when its name is in the
-// materialized-view set. Regular tables and views are left untouched.
+// emits SHOW CREATE MATERIALIZED VIEW for them. StarRocks reports MVs as 'VIEW', which
+// becomes 'MATERIALIZED VIEW' when the name is in the materialized-view set. Regular
+// tables and views are left untouched.
func markMaterializedViews(tables []*TableSchema, materializedViewNames map[string]bool) {
for _, tbl := range tables {
if !materializedViewNames[tbl.Name] {
@@ -371,7 +337,7 @@ func getTableStmt(db *sql.DB, dbName, tblName, tblType string) (string, error) {
createStmt = strings.TrimSuffix(createStmt, ";")
return fmt.Sprintf(viewStmtFmt, tblName, createStmt), nil
case materializedViewType:
- // For Doris materialized views, we use SHOW CREATE MATERIALIZED VIEW.
+ // For materialized views we use SHOW CREATE MATERIALIZED VIEW.
// This command returns 2 columns: materialized view name and create statement.
query := fmt.Sprintf("SHOW CREATE MATERIALIZED VIEW `%s`.`%s`;", dbName, tblName)
var createStmt, unused string
diff --git a/backend/plugin/db/starrocks/dump_routine_probe_test.go b/backend/plugin/db/starrocks/dump_routine_probe_test.go
index 092f1cfac5..77af0603d1 100644
--- a/backend/plugin/db/starrocks/dump_routine_probe_test.go
+++ b/backend/plugin/db/starrocks/dump_routine_probe_test.go
@@ -19,28 +19,23 @@ import (
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
)
-// BYT-10211: neither Doris nor StarRocks has MySQL stored routines or events, and Doris
-// kernels that dropped the PL/SQL grammar (SelectDB 4.0.12+/4.1.8+/5.0.5+, apache/doris#58700)
-// reject `SHOW FUNCTION STATUS` outright, which aborted the whole schema sync. The dump must
-// complete without ever sending the MySQL routine or event probes.
+// BYT-10211: StarRocks has no MySQL stored routines or events, and a kernel that rejects
+// `SHOW FUNCTION STATUS` outright aborted the whole schema sync. The dump must complete
+// without ever sending the MySQL routine or event probes.
func TestDumpSendsNoRoutineOrEventProbe(t *testing.T) {
- for _, engine := range []storepb.Engine{storepb.Engine_DORIS, storepb.Engine_STARROCKS} {
- t.Run(engine.String(), func(t *testing.T) {
- dumpRecorder.reset(plsqlStrippedCatalog(engine))
- sqlDB, err := sql.Open(dumpRecorderDriverName, "")
- require.NoError(t, err)
- defer sqlDB.Close()
-
- d := &Driver{dbType: engine, db: sqlDB, databaseName: "prod_tg"}
- var out bytes.Buffer
- require.NoError(t, d.Dump(context.Background(), &out, nil))
-
- require.Contains(t, out.String(), "CREATE TABLE `orders`")
- require.Contains(t, out.String(), "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`")
- for _, q := range dumpRecorder.recorded() {
- require.NotRegexp(t, forbiddenProbe, q)
- }
- })
+ dumpRecorder.reset(plsqlStrippedCatalog())
+ sqlDB, err := sql.Open(dumpRecorderDriverName, "")
+ require.NoError(t, err)
+ defer sqlDB.Close()
+
+ d := &Driver{dbType: storepb.Engine_STARROCKS, db: sqlDB, databaseName: "analytics"}
+ var out bytes.Buffer
+ require.NoError(t, d.Dump(context.Background(), &out, nil))
+
+ require.Contains(t, out.String(), "CREATE TABLE `orders`")
+ require.Contains(t, out.String(), "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`")
+ for _, q := range dumpRecorder.recorded() {
+ require.NotRegexp(t, forbiddenProbe, q)
}
}
@@ -50,17 +45,15 @@ var (
)
// plsqlStrippedCatalog answers the dump's table and view queries with the column shapes a
-// live Doris 4.1.3 returns (and the StarRocks materialized-view catalog in its own shape),
-// and rejects the routine probes with the exact error a PL/SQL-stripped kernel returns.
-func plsqlStrippedCatalog(engine storepb.Engine) func(string) ([]string, [][]driver.Value, error) {
+// live StarRocks returns, and rejects the routine probes with the exact error a kernel
+// without the routine grammar returns.
+func plsqlStrippedCatalog() func(string) ([]string, [][]driver.Value, error) {
return func(query string) ([]string, [][]driver.Value, error) {
switch {
case routineStatusProbe.MatchString(query):
return nil, nil, &mysql.MySQLError{Number: 1105, SQLState: [5]byte{'H', 'Y', '0', '0', '0'}, Message: "errCode = 2, detailMessage = no viable alternative at input 'SHOW FUNCTION'(line 1, pos 5)"}
case strings.Contains(query, "information_schema.TABLES"):
return []string{"TABLE_NAME", "TABLE_TYPE"}, [][]driver.Value{{"orders", "BASE TABLE"}, {"v_orders", "VIEW"}}, nil
- case strings.Contains(query, "mv_infos("):
- return []string{"Name"}, nil, nil
case strings.Contains(query, "information_schema.materialized_views"):
return []string{"TABLE_NAME", "REFRESH_TYPE"}, nil, nil
case strings.HasPrefix(query, "SHOW CREATE TABLE"):
@@ -70,7 +63,7 @@ func plsqlStrippedCatalog(engine storepb.Engine) func(string) ([]string, [][]dri
case strings.HasPrefix(query, "SHOW COLUMNS FROM"):
return []string{"Field", "Type", "Null", "Key", "Default", "Extra"}, [][]driver.Value{{"id", "int", "YES", "", nil, ""}}, nil
}
- return nil, nil, errors.Errorf("unexpected statement for %s: %q", engine, query)
+ return nil, nil, errors.Errorf("unexpected statement: %q", query)
}
}
diff --git a/backend/plugin/db/starrocks/dump_test.go b/backend/plugin/db/starrocks/dump_test.go
index b38d5b6c9c..fab8a200ca 100644
--- a/backend/plugin/db/starrocks/dump_test.go
+++ b/backend/plugin/db/starrocks/dump_test.go
@@ -14,17 +14,17 @@ func TestGetTemporaryView(t *testing.T) {
}
// BYT-9689: the dump path re-tags materialized-view rows so it emits
-// SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE',
-// StarRocks as 'VIEW'; both must become 'MATERIALIZED VIEW' when the name is a known
-// MV, while regular tables and views are left untouched.
+// SHOW CREATE MATERIALIZED VIEW for them. StarRocks reports MVs as 'VIEW', and some
+// setups as 'BASE TABLE'; either must become 'MATERIALIZED VIEW' when the name is a
+// known MV, while regular tables and views are left untouched.
func TestMarkMaterializedViews(t *testing.T) {
tables := []*TableSchema{
- {Name: "mv_doris", TableType: baseTableType}, // Doris MV reported as BASE TABLE
- {Name: "mv_starrocks", TableType: viewTableType}, // StarRocks MV reported as VIEW
- {Name: "v_regular", TableType: viewTableType}, // regular view (not an MV)
- {Name: "t_plain", TableType: baseTableType}, // plain table
+ {Name: "mv_view", TableType: viewTableType}, // MV reported as VIEW
+ {Name: "mv_base", TableType: baseTableType}, // MV reported as BASE TABLE
+ {Name: "v_regular", TableType: viewTableType},
+ {Name: "t_plain", TableType: baseTableType},
}
- markMaterializedViews(tables, map[string]bool{"mv_doris": true, "mv_starrocks": true})
+ markMaterializedViews(tables, map[string]bool{"mv_view": true, "mv_base": true})
require.Equal(t, materializedViewType, tables[0].TableType)
require.Equal(t, materializedViewType, tables[1].TableType)
diff --git a/backend/plugin/db/starrocks/starrocks.go b/backend/plugin/db/starrocks/starrocks.go
index 910b03b34a..645abfe2ed 100644
--- a/backend/plugin/db/starrocks/starrocks.go
+++ b/backend/plugin/db/starrocks/starrocks.go
@@ -11,7 +11,7 @@ import (
"strings"
"time"
- mysqlomni "github.com/bytebase/omni/mysql/parser"
+ starrocksomni "github.com/bytebase/omni/starrocks/parser"
"github.com/go-sql-driver/mysql"
"github.com/google/uuid"
"github.com/pkg/errors"
@@ -28,9 +28,7 @@ import (
"github.com/bytebase/bytebase/backend/plugin/db/util"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
- // Register how these engines plan a statement, for base.ExplainStatement below.
- // This driver serves both.
- _ "github.com/bytebase/bytebase/backend/plugin/parser/doris"
+ // Register how StarRocks plans a statement, for base.ExplainStatement below.
_ "github.com/bytebase/bytebase/backend/plugin/parser/starrocks"
)
@@ -44,10 +42,9 @@ var (
func init() {
db.Register(storepb.Engine_STARROCKS, newDriver)
- db.Register(storepb.Engine_DORIS, newDriver)
}
-// Driver is the MySQL driver.
+// Driver is the StarRocks driver.
type Driver struct {
dbType storepb.Engine
db *sql.DB
@@ -62,7 +59,7 @@ func newDriver() db.Driver {
return &Driver{}
}
-// Open opens a MySQL driver.
+// Open opens a StarRocks driver.
func (d *Driver) Open(_ context.Context, dbType storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {
defer func() {
for _, f := range d.openCleanUp {
@@ -74,10 +71,7 @@ func (d *Driver) Open(_ context.Context, dbType storepb.Engine, connCfg db.Conne
if strings.HasPrefix(connCfg.DataSource.Host, "/") {
protocol = "unix"
}
- params := []string{"multiStatements=true", "maxAllowedPacket=0"}
- if dbType == storepb.Engine_DORIS || dbType == storepb.Engine_STARROCKS {
- params = append(params, "interpolateParams=true")
- }
+ params := []string{"multiStatements=true", "maxAllowedPacket=0", "interpolateParams=true"}
if connCfg.DataSource.GetSshHost() != "" {
sshClient, err := util.GetSSHClient(connCfg.DataSource)
if err != nil {
@@ -177,7 +171,7 @@ func (d *Driver) Execute(ctx context.Context, statement string, opts db.ExecuteO
// Only preprocess when DELIMITER directives are present; normal
// StarRocks SQL passes through unchanged.
if containsDelimiterDirective(statement) {
- segments := mysqlomni.Split(statement)
+ segments := starrocksomni.Split(statement)
var buf strings.Builder
for i, seg := range segments {
if i > 0 {
diff --git a/backend/plugin/db/starrocks/starrocks_test.go b/backend/plugin/db/starrocks/starrocks_test.go
index d84a8c1fdc..f6ef7b302f 100644
--- a/backend/plugin/db/starrocks/starrocks_test.go
+++ b/backend/plugin/db/starrocks/starrocks_test.go
@@ -8,8 +8,7 @@ import (
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
- // Register the doris + starrocks editor validators.
- _ "github.com/bytebase/bytebase/backend/plugin/parser/doris"
+ // Register the StarRocks editor validator.
_ "github.com/bytebase/bytebase/backend/plugin/parser/starrocks"
)
@@ -72,19 +71,13 @@ func TestContainsDelimiterDirective(t *testing.T) {
}
}
-// TestValidateSQLForEditor_EngineDispatch guards the #20562 P2 fix: QueryConn
-// must split/validate via the connection's engine (d.dbType), not a hardcoded
-// Engine_DORIS. On a StarRocks-only statement the two engines must disagree —
-// doris rejects it (→ allQuery forced true, DDL wrongly routed through
-// QueryContext), starrocks accepts it (→ correct allQuery=false).
-func TestValidateSQLForEditor_EngineDispatch(t *testing.T) {
- // StarRocks generated column AS (expr): doris wants GENERATED ALWAYS AS.
- const ddl = "CREATE TABLE t (a INT, b INT AS (a + 1)) DUPLICATE KEY(a) DISTRIBUTED BY HASH(a)"
-
- _, srAllQuery, srErr := base.ValidateSQLForEditor(storepb.Engine_STARROCKS, ddl)
- require.NoError(t, srErr, "starrocks parser should accept StarRocks DDL")
- require.False(t, srAllQuery, "a CREATE TABLE is DDL, not a read-only query")
-
- _, _, dorisErr := base.ValidateSQLForEditor(storepb.Engine_DORIS, ddl)
- require.Error(t, dorisErr, "doris parser rejects StarRocks AS(expr) DDL — why d.dbType matters")
+// TestValidateSQLForEditorUsesStarRocksGrammar pins this driver to the StarRocks
+// parser: a StarRocks-only spelling must be accepted, which routing the connection
+// through a sibling engine's grammar would not do.
+func TestValidateSQLForEditorUsesStarRocksGrammar(t *testing.T) {
+ // StarRocks generated column AS (expr); Doris wants GENERATED ALWAYS AS.
+ const starRocksOnlyDDL = "CREATE TABLE t (a INT, b INT AS (a + 1)) DUPLICATE KEY(a) DISTRIBUTED BY HASH(a)"
+ _, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_STARROCKS, starRocksOnlyDDL)
+ require.NoError(t, err)
+ require.False(t, allQuery, "a CREATE TABLE is DDL, not a read-only query")
}
diff --git a/backend/plugin/db/starrocks/sync.go b/backend/plugin/db/starrocks/sync.go
index 510f27bc8b..dd9cbbd128 100644
--- a/backend/plugin/db/starrocks/sync.go
+++ b/backend/plugin/db/starrocks/sync.go
@@ -24,15 +24,10 @@ const (
)
// systemDatabaseExclusion returns the quoted, comma-joined system databases to
-// exclude from instance discovery, per the driver's engine. StarRocks adds its
-// read-only `sys` metadatabase (Doris has no sys), matching the parser-side
-// system-database classification.
-func (d *Driver) systemDatabaseExclusion() string {
- dbs := []string{"'information_schema'", "'_statistics_'"}
- if d.dbType == storepb.Engine_STARROCKS {
- dbs = append(dbs, "'sys'")
- }
- return strings.Join(dbs, ", ")
+// exclude from instance discovery, matching the parser-side system-database
+// classification. `sys` is StarRocks's read-only metadatabase, not a user database.
+func systemDatabaseExclusion() string {
+ return strings.Join([]string{"'information_schema'", "'_statistics_'", "'sys'"}, ", ")
}
// SyncInstance syncs the instance.
@@ -56,7 +51,7 @@ func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error)
}
// Query db info
- where := fmt.Sprintf("LOWER(SCHEMA_NAME) NOT IN (%s)", d.systemDatabaseExclusion())
+ where := fmt.Sprintf("LOWER(SCHEMA_NAME) NOT IN (%s)", systemDatabaseExclusion())
query := `
SELECT
SCHEMA_NAME,
@@ -80,10 +75,8 @@ func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error)
); err != nil {
return nil, err
}
- if d.dbType == storepb.Engine_DORIS || d.dbType == storepb.Engine_STARROCKS {
- database.CharacterSet = ""
- database.Collation = ""
- }
+ database.CharacterSet = ""
+ database.Collation = ""
databases = append(databases, database)
}
if err := rows.Err(); err != nil {
@@ -127,9 +120,8 @@ func isSyncRollup(refreshType string) bool {
// isMaterializedView reports whether a table row is a materialized view, based on its
// table type and membership in the materialized-view set. StarRocks reports
-// materialized views with TABLE_TYPE='VIEW', Doris with 'BASE TABLE', and some setups
-// with 'MATERIALIZED VIEW'; in every case, presence in materializedViewMap is the
-// authoritative signal.
+// materialized views with TABLE_TYPE='VIEW', and some setups with 'MATERIALIZED VIEW';
+// in either case, presence in materializedViewMap is the authoritative signal.
func isMaterializedView(tableType string, key db.TableKey, materializedViewMap map[db.TableKey]*metadatapb.MaterializedViewMetadata) bool {
if _, ok := materializedViewMap[key]; !ok {
return false
@@ -143,26 +135,15 @@ func isMaterializedView(tableType string, key db.TableKey, materializedViewMap m
}
// getMaterializedViews returns the materialized views of the current database keyed by
-// table name. Doris and StarRocks expose them through different catalogs, so the query
-// and projection are engine-specific. The query is best-effort: an engine/version that
-// does not support it is logged and yields an empty set rather than failing the sync.
+// table name. The query is best-effort: a version that does not support it is logged
+// and yields an empty set rather than failing the sync.
func (d *Driver) getMaterializedViews(ctx context.Context) (map[db.TableKey]*metadatapb.MaterializedViewMetadata, error) {
materializedViewMap := make(map[db.TableKey]*metadatapb.MaterializedViewMetadata)
- var query string
- switch d.dbType {
- case storepb.Engine_DORIS:
- // Doris exposes materialized views via the mv_infos() table-valued function.
- // https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/mv_infos
- query = fmt.Sprintf(`SELECT Name, QuerySql FROM mv_infos("database"="%s")`, d.databaseName)
- case storepb.Engine_STARROCKS:
- // StarRocks exposes materialized views via information_schema.materialized_views.
- // REFRESH_TYPE separates async MVs from synchronous rollups, which are excluded.
- // IFNULL guards the bare-string scans below: database/sql errors on a NULL->string scan.
- query = fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, ''), IFNULL(MATERIALIZED_VIEW_DEFINITION, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, d.databaseName)
- default:
- return materializedViewMap, nil
- }
+ // StarRocks exposes materialized views via information_schema.materialized_views.
+ // REFRESH_TYPE separates async MVs from synchronous rollups, which are excluded.
+ // IFNULL guards the bare-string scans below: database/sql errors on a NULL->string scan.
+ query := fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, ''), IFNULL(MATERIALIZED_VIEW_DEFINITION, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, d.databaseName)
rows, err := d.db.QueryContext(ctx, query)
if err != nil {
@@ -174,17 +155,13 @@ func (d *Driver) getMaterializedViews(ctx context.Context) (map[db.TableKey]*met
for rows.Next() {
materializedView := &metadatapb.MaterializedViewMetadata{}
- if d.dbType == storepb.Engine_STARROCKS {
- var refreshType string
- if err := rows.Scan(&materializedView.Name, &refreshType, &materializedView.Definition); err != nil {
- return nil, err
- }
- if isSyncRollup(refreshType) {
- continue
- }
- } else if err := rows.Scan(&materializedView.Name, &materializedView.Definition); err != nil {
+ var refreshType string
+ if err := rows.Scan(&materializedView.Name, &refreshType, &materializedView.Definition); err != nil {
return nil, err
}
+ if isSyncRollup(refreshType) {
+ continue
+ }
key := db.TableKey{Schema: "", Table: materializedView.Name}
materializedViewMap[key] = materializedView
}
@@ -354,7 +331,7 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
}
key := db.TableKey{Schema: "", Table: tableName}
- // StarRocks reports materialized views as TABLE_TYPE='VIEW', Doris as 'BASE TABLE';
+ // StarRocks reports materialized views as TABLE_TYPE='VIEW';
// route any row whose name is in the materialized-view set to MaterializedViews.
if isMaterializedView(tableType, key, materializedViewMap) {
materializedView := materializedViewMap[key]
@@ -418,10 +395,8 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe
}
// "characterSet":"utf8\u0000", "collation":"utf8_general_ci\u0000".
// ERROR: unsupported Unicode escape sequence (SQLSTATE 22P05).
- if d.dbType == storepb.Engine_DORIS || d.dbType == storepb.Engine_STARROCKS {
- databaseMetadata.CharacterSet = ""
- databaseMetadata.Collation = ""
- }
+ databaseMetadata.CharacterSet = ""
+ databaseMetadata.Collation = ""
return databaseMetadata, err
}
diff --git a/backend/plugin/db/starrocks/sync_test.go b/backend/plugin/db/starrocks/sync_test.go
index 8a823b1e0d..ef4a00df64 100644
--- a/backend/plugin/db/starrocks/sync_test.go
+++ b/backend/plugin/db/starrocks/sync_test.go
@@ -6,25 +6,22 @@ import (
metadatapb "github.com/bytebase/omni/metadata"
"github.com/stretchr/testify/require"
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/db"
)
// BYT-9689: StarRocks reports materialized views in information_schema.tables with
// TABLE_TYPE='VIEW' and omits them from information_schema.views (verified on a live
-// 3.4.10 BE-alive smoke test), while Doris reports them as 'BASE TABLE'. The shared
-// driver previously only matched Doris's 'BASE TABLE' rows, so StarRocks MVs were
-// dropped. Membership in the materialized-view set is the authoritative signal.
+// 3.4.10 BE-alive smoke test), so the table type alone cannot identify one. Membership
+// in the materialized-view set is the authoritative signal, whatever type the row carries.
func TestIsMaterializedView(t *testing.T) {
mvMap := map[db.TableKey]*metadatapb.MaterializedViewMetadata{
{Schema: "", Table: "mv_async"}: {Name: "mv_async"},
}
mvKey := db.TableKey{Schema: "", Table: "mv_async"}
- // StarRocks reports the MV as VIEW; Doris as BASE TABLE; both map to an MV.
require.True(t, isMaterializedView(viewTableType, mvKey, mvMap))
- require.True(t, isMaterializedView(baseTableType, mvKey, mvMap))
require.True(t, isMaterializedView(materializedViewType, mvKey, mvMap))
+ require.True(t, isMaterializedView(baseTableType, mvKey, mvMap))
// A name absent from the MV set is never a materialized view.
require.False(t, isMaterializedView(viewTableType, db.TableKey{Table: "v_regular"}, mvMap))
@@ -45,17 +42,12 @@ func TestIsSyncRollup(t *testing.T) {
require.False(t, isSyncRollup("")) // IFNULL-coalesced NULL or a genuinely empty type.
}
-// #20566 review (P2): SyncInstance's system-database exclusion must match the
-// parser's per-engine system set — StarRocks's read-only `sys` metadatabase
-// must be excluded so it isn't imported as a user database. Doris has no `sys`,
-// so excluding it there would risk hiding a user db named sys (engine-aware).
+// #20566 review (P2): SyncInstance's system-database exclusion must match the parser's
+// system set. `sys` is StarRocks's read-only metadatabase and must be excluded so it
+// isn't imported as a user database.
func TestSystemDatabaseExclusion(t *testing.T) {
- sr := (&Driver{dbType: storepb.Engine_STARROCKS}).systemDatabaseExclusion()
- require.Contains(t, sr, "'sys'", "StarRocks sync must exclude the sys metadatabase")
- require.Contains(t, sr, "'information_schema'")
- require.Contains(t, sr, "'_statistics_'")
-
- doris := (&Driver{dbType: storepb.Engine_DORIS}).systemDatabaseExclusion()
- require.NotContains(t, doris, "'sys'", "Doris must not exclude sys (StarRocks-only)")
- require.Contains(t, doris, "'information_schema'")
+ got := systemDatabaseExclusion()
+ require.Contains(t, got, "'information_schema'")
+ require.Contains(t, got, "'_statistics_'")
+ require.Contains(t, got, "'sys'")
}
diff --git a/backend/plugin/parser/doris/doris.go b/backend/plugin/parser/doris/doris.go
index 7bd674f1f6..b13b78b0c9 100644
--- a/backend/plugin/parser/doris/doris.go
+++ b/backend/plugin/parser/doris/doris.go
@@ -19,6 +19,7 @@ func init() {
// omniAST wraps an omni AST node to implement the base.AST interface.
type omniAST struct {
node ast.Node
+ text string
startPos *storepb.Position
}
@@ -67,6 +68,7 @@ func parseDorisStatements(statement string) ([]base.ParsedStatement, error) {
}
ps.AST = &omniAST{
node: node,
+ text: stmt.Text,
startPos: stmt.Start,
}
result = append(result, ps)
@@ -103,6 +105,7 @@ func parseDorisSQL(statement string) ([]*omniAST, error) {
}
result = append(result, &omniAST{
node: node,
+ text: stmt.Text,
startPos: stmt.Start,
})
}
diff --git a/backend/plugin/parser/doris/resource_change.go b/backend/plugin/parser/doris/resource_change.go
new file mode 100644
index 0000000000..a166bd1d5d
--- /dev/null
+++ b/backend/plugin/parser/doris/resource_change.go
@@ -0,0 +1,161 @@
+package doris
+
+import (
+ "strings"
+
+ "github.com/bytebase/omni/doris/ast"
+ "github.com/pkg/errors"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store/model"
+)
+
+func init() {
+ base.RegisterExtractChangedResourcesFunc(storepb.Engine_DORIS, extractChangedResources)
+}
+
+func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {
+ changedResources := model.NewChangedResources(dbMetadata)
+
+ var dmlCount, insertCount int
+ var dmlStatements []string
+
+ addTable := func(name *ast.ObjectName, affectsExistingRows bool) {
+ db, table := objectName(name, currentDatabase)
+ if table == "" {
+ return
+ }
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, affectsExistingRows)
+ }
+
+ // addObjectDatabase records a database-only write target for a non-table object DDL
+ // (view, materialized view, routine) ONLY when the object's own name carries an
+ // explicit database qualifier. An unqualified name records nothing, leaving the
+ // request database to stand in. SUP-222.
+ addObjectDatabase := func(name *ast.ObjectName) {
+ if db, _ := objectName(name, ""); db != "" {
+ changedResources.AddDatabase(db)
+ }
+ }
+
+ for _, unifiedAST := range asts {
+ parsed, ok := unifiedAST.(*omniAST)
+ if !ok {
+ return nil, errors.New("expected omniAST for Doris")
+ }
+ if parsed.node == nil {
+ continue
+ }
+
+ switch n := parsed.node.(type) {
+ case *ast.CreateTableStmt:
+ addTable(n.Name, false)
+
+ case *ast.DropTableStmt:
+ addTable(n.Name, true)
+
+ case *ast.TruncateTableStmt:
+ addTable(n.Target, true)
+
+ case *ast.AlterTableStmt:
+ addTable(n.Name, true)
+
+ case *ast.CreateIndexStmt:
+ addTable(n.Table, false)
+
+ case *ast.DropIndexStmt:
+ addTable(n.Table, false)
+
+ case *ast.LoadDataStmt:
+ // One LOAD LABEL carries a data description per target table.
+ for _, desc := range n.DataDescs {
+ if desc != nil {
+ addTable(desc.Target, false)
+ }
+ }
+
+ case *ast.InsertStmt:
+ addTable(n.Target, false)
+ if len(n.Values) > 0 {
+ insertCount += len(n.Values)
+ continue
+ }
+ dmlCount++
+ dmlStatements = append(dmlStatements, statementText(parsed))
+
+ case *ast.UpdateStmt:
+ // Doris updates a single target; FROM names read sources, not write targets.
+ addTable(n.Target, false)
+ dmlCount++
+ dmlStatements = append(dmlStatements, statementText(parsed))
+
+ case *ast.DeleteStmt:
+ // USING names read sources, as FROM does for UPDATE.
+ addTable(n.Target, false)
+ dmlCount++
+ dmlStatements = append(dmlStatements, statementText(parsed))
+
+ case *ast.MergeStmt:
+ addTable(n.Target, false)
+ dmlCount++
+ dmlStatements = append(dmlStatements, statementText(parsed))
+
+ // Non-table object DDL: gate by the object's own explicit database qualifier only.
+ // CREATE FUNCTION is absent because its AST keeps the name inside RawSignature
+ // rather than as an ObjectName, so there is nothing to qualify by.
+ case *ast.CreateViewStmt:
+ addObjectDatabase(n.Name)
+ case *ast.AlterViewStmt:
+ addObjectDatabase(n.Name)
+ case *ast.DropViewStmt:
+ addObjectDatabase(n.Name)
+ case *ast.CreateMTMVStmt:
+ addObjectDatabase(n.Name)
+ case *ast.AlterMTMVStmt:
+ addObjectDatabase(n.Name)
+ case *ast.DropMTMVStmt:
+ addObjectDatabase(n.Name)
+ case *ast.DropFunctionStmt:
+ addObjectDatabase(n.Name)
+ case *ast.CreateProcedureStmt:
+ addObjectDatabase(n.Name)
+ case *ast.DropProcedureStmt:
+ addObjectDatabase(n.Name)
+
+ default:
+ }
+ }
+
+ return &base.ChangeSummary{
+ ChangedResources: changedResources,
+ DMLStatements: dmlStatements,
+ DMLCount: dmlCount,
+ InsertCount: insertCount,
+ }, nil
+}
+
+// objectName splits an ObjectName into its database and object name. Doris writes a
+// name as table, db.table or catalog.db.table, so the object is the last part and the
+// database the one before it; a catalog qualifier is dropped, since changed resources
+// are tracked per database.
+func objectName(name *ast.ObjectName, defaultDatabase string) (string, string) {
+ if name == nil || len(name.Parts) == 0 {
+ return defaultDatabase, ""
+ }
+ parts := name.Parts
+ database := defaultDatabase
+ if len(parts) >= 2 {
+ database = parts[len(parts)-2]
+ }
+ return database, parts[len(parts)-1]
+}
+
+// statementText returns the text of a statement, ensuring it ends with a semicolon.
+func statementText(a *omniAST) string {
+ text := strings.TrimSpace(a.text)
+ if !strings.HasSuffix(text, ";") {
+ text += ";"
+ }
+ return text
+}
diff --git a/backend/plugin/parser/doris/resource_change_test.go b/backend/plugin/parser/doris/resource_change_test.go
new file mode 100644
index 0000000000..8383ccd8d8
--- /dev/null
+++ b/backend/plugin/parser/doris/resource_change_test.go
@@ -0,0 +1,149 @@
+package doris
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store/model"
+)
+
+// changedTables flattens the built resources into "database.table" strings.
+func changedTables(t *testing.T, statement string) []string {
+ t.Helper()
+ summary := extract(t, statement, nil)
+ var result []string
+ for _, database := range summary.ChangedResources.Build().GetDatabases() {
+ for _, schema := range database.GetSchemas() {
+ for _, table := range schema.GetTables() {
+ result = append(result, database.GetName()+"."+table.GetName())
+ }
+ }
+ }
+ return result
+}
+
+func extract(t *testing.T, statement string, dbMetadata *model.DatabaseMetadata) *base.ChangeSummary {
+ t.Helper()
+ stmts, err := base.ParseStatements(storepb.Engine_DORIS, statement)
+ require.NoError(t, err)
+ summary, err := base.ExtractChangedResources(storepb.Engine_DORIS, "db", "", dbMetadata, base.ExtractASTs(stmts), statement)
+ require.NoError(t, err)
+ return summary
+}
+
+func TestExtractChangedResourcesTables(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ want []string
+ }{
+ {"create table takes the request database", "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"db.t"}},
+ {"a qualified name keeps its own database", "CREATE TABLE other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"other.t"}},
+ {"a catalog qualifier is dropped", "CREATE TABLE cat.other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"other.t"}},
+ {"drop table", "DROP TABLE t;", []string{"db.t"}},
+ {"truncate table", "TRUNCATE TABLE t;", []string{"db.t"}},
+ {"alter table", "ALTER TABLE t ADD COLUMN b INT;", []string{"db.t"}},
+ {"create index names its table", "CREATE INDEX idx ON t (a);", []string{"db.t"}},
+ {"drop index names its table", "DROP INDEX idx ON t;", []string{"db.t"}},
+ {"insert names its target", "INSERT INTO t VALUES (1);", []string{"db.t"}},
+ {"update names its target, not its source", "UPDATE t SET a = 1 WHERE a IN (SELECT a FROM s);", []string{"db.t"}},
+ {"delete names its target, not its source", "DELETE FROM t WHERE a = 1;", []string{"db.t"}},
+ {"a view records no table", "CREATE VIEW v AS SELECT 1;", nil},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ require.ElementsMatch(t, tc.want, changedTables(t, tc.statement))
+ })
+ }
+}
+
+func TestExtractChangedResourcesDMLAccounting(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ wantDML int
+ wantInserts int
+ wantTexts []string
+ }{
+ {name: "insert values counts rows, not statements", statement: "INSERT INTO t VALUES (1), (2), (3);", wantInserts: 3},
+ {name: "insert select is a DML statement", statement: "INSERT INTO t SELECT a FROM s;", wantDML: 1, wantTexts: []string{"INSERT INTO t SELECT a FROM s;"}},
+ {name: "update is a DML statement", statement: "UPDATE t SET a = 1 WHERE a = 2;", wantDML: 1, wantTexts: []string{"UPDATE t SET a = 1 WHERE a = 2;"}},
+ {name: "delete is a DML statement", statement: "DELETE FROM t WHERE a = 1;", wantDML: 1, wantTexts: []string{"DELETE FROM t WHERE a = 1;"}},
+ {name: "DDL is neither", statement: "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;"},
+ {
+ name: "a script accumulates both",
+ statement: "INSERT INTO t VALUES (1), (2);\nUPDATE t SET a = 1;\nDELETE FROM t;",
+ wantDML: 2,
+ wantInserts: 2,
+ wantTexts: []string{"UPDATE t SET a = 1;", "DELETE FROM t;"},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ summary := extract(t, tc.statement, nil)
+ require.Equal(t, tc.wantDML, summary.DMLCount)
+ require.Equal(t, tc.wantInserts, summary.InsertCount)
+ require.Equal(t, tc.wantTexts, summary.DMLStatements)
+ })
+ }
+}
+
+func TestExtractChangedResourcesDatabaseOnlyTargets(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ want []string
+ }{
+ {"a qualified view records its database", "CREATE VIEW other.v AS SELECT 1;", []string{"other"}},
+ {"an unqualified view records nothing", "CREATE VIEW v AS SELECT 1;", []string{}},
+ {"alter view", "ALTER VIEW other.v AS SELECT 1;", []string{"other"}},
+ {"drop view", "DROP VIEW other.v;", []string{"other"}},
+ {"a qualified materialized view records its database", "DROP MATERIALIZED VIEW other.mv;", []string{"other"}},
+ {"drop procedure", "DROP PROCEDURE other.p;", []string{"other"}},
+ {"a table DDL records no database-only target", "CREATE TABLE other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ summary := extract(t, tc.statement, nil)
+ require.Equal(t, tc.want, summary.ChangedResources.GetDatabaseOnlyTargets())
+ })
+ }
+}
+
+func TestExtractChangedResourcesAffectedRows(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{Tables: []*metadatapb.TableMetadata{{Name: "t", RowCount: 42}}}},
+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_DORIS, false /* caseSensitive */)
+
+ for _, tc := range []struct {
+ name string
+ statement string
+ want int64
+ }{
+ {"drop discards existing rows", "DROP TABLE t;", 42},
+ {"truncate discards existing rows", "TRUNCATE TABLE t;", 42},
+ {"alter rewrites existing rows", "ALTER TABLE t ADD COLUMN b INT;", 42},
+ {"create has no existing rows", "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", 0},
+ {"insert leaves existing rows alone", "INSERT INTO t VALUES (1);", 0},
+ {"create index leaves existing rows alone", "CREATE INDEX idx ON t (a);", 0},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ summary := extract(t, tc.statement, dbMetadata)
+ require.Equal(t, tc.want, summary.ChangedResources.CountAffectedTableRows())
+ })
+ }
+}
+
+// foreignAST stands in for another engine's AST, which reached this extractor
+// through the shared registry before Doris had one of its own.
+type foreignAST struct{}
+
+func (*foreignAST) ASTStartPosition() *storepb.Position { return nil }
+
+func TestExtractChangedResourcesRejectsForeignAST(t *testing.T) {
+ _, err := extractChangedResources("db", "", nil, []base.AST{&foreignAST{}}, "")
+ require.ErrorContains(t, err, "expected omniAST for Doris")
+}
diff --git a/backend/plugin/parser/mysql/query_span_extractor.go b/backend/plugin/parser/mysql/query_span_extractor.go
index ca4a6a1326..da9f4579dc 100644
--- a/backend/plugin/parser/mysql/query_span_extractor.go
+++ b/backend/plugin/parser/mysql/query_span_extractor.go
@@ -10,7 +10,6 @@ import (
"github.com/bytebase/omni/mysql/ast"
"github.com/pkg/errors"
- storepb "github.com/bytebase/bytebase/backend/generated-go/store"
"github.com/bytebase/bytebase/backend/plugin/parser/base"
"github.com/bytebase/bytebase/backend/store/model"
)
@@ -162,7 +161,6 @@ func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName st
// SELECT first, by the rules of the clause the reference is in, then each
// enclosing SELECT outward, its tables before the select list it exposes.
func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldName string) (base.SourceColumnSet, error) {
- databaseName = q.filterClusterName(databaseName)
if sourceColumnSet, ok := q.resolveInCurrentSelect(databaseName, tableName, fieldName); ok {
return sourceColumnSet, nil
}
@@ -229,16 +227,6 @@ func (q *querySpanExtractor) findColumnInTables(tables []base.TableSource, datab
return nil, false
}
-func (q *querySpanExtractor) filterClusterName(databaseName string) string {
- if q.gCtx.Engine == storepb.Engine_STARROCKS {
- list := strings.Split(databaseName, ":")
- if len(list) > 1 {
- databaseName = list[len(list)-1]
- }
- }
- return databaseName
-}
-
func (q *querySpanExtractor) findTableSchema(databaseName, tableName string) (base.TableSource, error) {
if databaseName == "" {
for i := len(q.ctes) - 1; i >= 0; i-- {
@@ -253,7 +241,6 @@ func (q *querySpanExtractor) findTableSchema(databaseName, tableName string) (ba
if databaseName == "" {
databaseName = q.defaultDatabase
}
- databaseName = q.filterClusterName(databaseName)
var dbMetadata *model.DatabaseMetadata
allDatabaseNames, err := q.gCtx.ListDatabaseNamesFunc(q.ctx, q.gCtx.InstanceID)
@@ -456,7 +443,7 @@ func (q *omniQuerySpanExtractor) getOmniQuerySpan(ctx context.Context, statement
}
root := stmts.Items[0]
- accessTables := collectOmniAccessTables(root, q.defaultDatabase, q.gCtx.Engine == storepb.Engine_STARROCKS)
+ accessTables := collectOmniAccessTables(root, q.defaultDatabase)
allSystems, mixed := isMixedQuery(accessTables, q.ignoreCaseSensitive)
if mixed {
return nil, base.MixUserSystemTablesError
@@ -1635,13 +1622,13 @@ func isOmniDescribeStatement(source string, loc ast.Loc) bool {
return len(fields) > 0 && (fields[0] == "DESCRIBE" || fields[0] == "DESC")
}
-func collectOmniAccessTables(root ast.Node, defaultDatabase string, normalizeStarRocksCluster bool) base.SourceColumnSet {
+func collectOmniAccessTables(root ast.Node, defaultDatabase string) base.SourceColumnSet {
result := make(base.SourceColumnSet)
- collectOmniAccessTablesFromNode(result, root, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, root, defaultDatabase)
return result
}
-func collectOmniAccessTablesFromNode(result base.SourceColumnSet, node ast.Node, defaultDatabase string, normalizeStarRocksCluster bool) {
+func collectOmniAccessTablesFromNode(result base.SourceColumnSet, node ast.Node, defaultDatabase string) {
switch n := node.(type) {
case nil:
case *ast.SelectStmt:
@@ -1650,206 +1637,206 @@ func collectOmniAccessTablesFromNode(result base.SourceColumnSet, node ast.Node,
}
for _, cte := range n.CTEs {
if cte != nil {
- collectOmniAccessTablesFromNode(result, cte.Select, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, cte.Select, defaultDatabase)
}
}
if n.Left != nil {
- collectOmniAccessTablesFromNode(result, n.Left, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, n.Left, defaultDatabase)
}
if n.Right != nil {
- collectOmniAccessTablesFromNode(result, n.Right, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, n.Right, defaultDatabase)
}
if inner := n.ParenSource; inner != nil {
- collectOmniAccessTablesFromNode(result, inner, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, inner, defaultDatabase)
}
if tableStmt := n.TableSource; tableStmt != nil {
- collectOmniAccessTablesFromNode(result, tableStmt, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, tableStmt, defaultDatabase)
}
if valuesStmt := n.ValuesSource; valuesStmt != nil {
- collectOmniAccessTablesFromNode(result, valuesStmt, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, valuesStmt, defaultDatabase)
}
for _, target := range n.TargetList {
- collectOmniAccessTablesFromExpr(result, target, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, target, defaultDatabase)
}
for _, tableExpr := range n.From {
- collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase)
}
- collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase)
for _, groupBy := range n.GroupBy {
- collectOmniAccessTablesFromExpr(result, groupBy, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, groupBy, defaultDatabase)
}
- collectOmniAccessTablesFromExpr(result, n.Having, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, n.Having, defaultDatabase)
for _, orderBy := range n.OrderBy {
if orderBy != nil {
- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)
}
}
case *ast.ExplainStmt:
- collectOmniAccessTablesFromNode(result, n.Stmt, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, n.Stmt, defaultDatabase)
case *ast.InsertStmt:
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
for _, row := range n.Values {
for _, expr := range row {
- collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase)
}
}
- collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromNode(result, n.TableSource, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase)
+ collectOmniAccessTablesFromNode(result, n.TableSource, defaultDatabase)
for _, assignment := range n.SetList {
if assignment != nil {
- collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase)
}
}
for _, assignment := range n.OnDuplicateKey {
if assignment != nil {
- collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase)
}
}
case *ast.UpdateStmt:
for _, tableExpr := range n.Tables {
- collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase)
}
for _, assignment := range n.SetList {
if assignment != nil {
- collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase)
}
}
- collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase)
for _, orderBy := range n.OrderBy {
if orderBy != nil {
- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)
}
}
case *ast.DeleteStmt:
for _, tableExpr := range n.Tables {
- collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase)
}
for _, tableExpr := range n.Using {
- collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase)
}
- collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase)
for _, orderBy := range n.OrderBy {
if orderBy != nil {
- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)
}
}
case *ast.CreateTableStmt:
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromTableRef(result, n.Like, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
+ collectOmniAccessTablesFromTableRef(result, n.Like, defaultDatabase)
+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase)
case *ast.CreateIndexStmt:
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
for _, column := range n.Columns {
if column != nil {
- collectOmniAccessTablesFromExpr(result, column.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, column.Expr, defaultDatabase)
}
}
case *ast.CreateViewStmt:
- collectOmniAccessTablesFromTableRef(result, n.Name, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Name, defaultDatabase)
+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase)
case *ast.AlterTableStmt:
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
for _, command := range n.Commands {
if command == nil {
continue
}
- collectOmniAccessTablesFromExpr(result, command.DefaultExpr, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromTableRef(result, command.ExchangeTable, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromOrderBy(result, command.OrderByItems, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, command.DefaultExpr, defaultDatabase)
+ collectOmniAccessTablesFromTableRef(result, command.ExchangeTable, defaultDatabase)
+ collectOmniAccessTablesFromOrderBy(result, command.OrderByItems, defaultDatabase)
}
case *ast.DropTableStmt:
- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)
case *ast.DropIndexStmt:
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
case *ast.DropViewStmt:
- collectOmniAccessTablesFromTableRefs(result, n.Views, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRefs(result, n.Views, defaultDatabase)
case *ast.RenameTableStmt:
for _, pair := range n.Pairs {
if pair == nil {
continue
}
- collectOmniAccessTablesFromTableRef(result, pair.Old, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromTableRef(result, pair.New, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, pair.Old, defaultDatabase)
+ collectOmniAccessTablesFromTableRef(result, pair.New, defaultDatabase)
}
case *ast.TruncateStmt:
- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)
case *ast.AnalyzeTableStmt:
- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)
case *ast.OptimizeTableStmt:
- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)
case *ast.CheckTableStmt:
- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)
case *ast.RepairTableStmt:
- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)
case *ast.TableStmt:
if n == nil {
return
}
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
+ collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase)
+ collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase)
case *ast.ValuesStmt:
for _, row := range n.Rows {
for _, expr := range row {
- collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase)
}
}
- collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase)
+ collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase)
case *ast.CallStmt:
for _, arg := range n.Args {
- collectOmniAccessTablesFromExpr(result, arg, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, arg, defaultDatabase)
}
case *ast.SetStmt:
for _, assignment := range n.Assignments {
if assignment != nil {
- collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase)
}
}
case *ast.DoStmt:
for _, expr := range n.Exprs {
- collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase)
}
case *ast.HandlerOpenStmt:
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
case *ast.HandlerReadStmt:
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase)
case *ast.HandlerCloseStmt:
- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)
case *ast.LockTablesStmt:
for _, table := range n.Tables {
if table != nil {
- collectOmniAccessTablesFromTableRef(result, table.Table, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, table.Table, defaultDatabase)
}
}
default:
}
}
-func collectOmniAccessTablesFromOrderBy(result base.SourceColumnSet, orderByItems []*ast.OrderByItem, defaultDatabase string, normalizeStarRocksCluster bool) {
+func collectOmniAccessTablesFromOrderBy(result base.SourceColumnSet, orderByItems []*ast.OrderByItem, defaultDatabase string) {
for _, orderBy := range orderByItems {
if orderBy != nil {
- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)
}
}
}
-func collectOmniAccessTablesFromLimit(result base.SourceColumnSet, limit *ast.Limit, defaultDatabase string, normalizeStarRocksCluster bool) {
+func collectOmniAccessTablesFromLimit(result base.SourceColumnSet, limit *ast.Limit, defaultDatabase string) {
if limit == nil {
return
}
- collectOmniAccessTablesFromExpr(result, limit.Count, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, limit.Offset, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, limit.Count, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, limit.Offset, defaultDatabase)
}
-func collectOmniAccessTablesFromTableRefs(result base.SourceColumnSet, tableRefs []*ast.TableRef, defaultDatabase string, normalizeStarRocksCluster bool) {
+func collectOmniAccessTablesFromTableRefs(result base.SourceColumnSet, tableRefs []*ast.TableRef, defaultDatabase string) {
for _, tableRef := range tableRefs {
- collectOmniAccessTablesFromTableRef(result, tableRef, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, tableRef, defaultDatabase)
}
}
-func collectOmniAccessTablesFromTableRef(result base.SourceColumnSet, tableRef *ast.TableRef, defaultDatabase string, normalizeStarRocksCluster bool) {
+func collectOmniAccessTablesFromTableRef(result base.SourceColumnSet, tableRef *ast.TableRef, defaultDatabase string) {
if tableRef == nil {
return
}
@@ -1859,8 +1846,6 @@ func collectOmniAccessTablesFromTableRef(result base.SourceColumnSet, tableRef *
database := tableRef.Schema
if database == "" {
database = defaultDatabase
- } else if normalizeStarRocksCluster {
- database = filterStarRocksClusterName(database)
}
result[base.ColumnResource{
Database: database,
@@ -1872,121 +1857,113 @@ func isOmniDualTable(tableRef *ast.TableRef) bool {
return tableRef != nil && tableRef.Schema == "" && strings.EqualFold(tableRef.Name, "dual")
}
-func collectOmniAccessTablesFromTableExpr(result base.SourceColumnSet, tableExpr ast.TableExpr, defaultDatabase string, normalizeStarRocksCluster bool) {
+func collectOmniAccessTablesFromTableExpr(result base.SourceColumnSet, tableExpr ast.TableExpr, defaultDatabase string) {
switch n := tableExpr.(type) {
case *ast.TableRef:
- collectOmniAccessTablesFromTableRef(result, n, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableRef(result, n, defaultDatabase)
case *ast.JoinClause:
- collectOmniAccessTablesFromTableExpr(result, n.Left, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromTableExpr(result, n.Right, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromTableExpr(result, n.Left, defaultDatabase)
+ collectOmniAccessTablesFromTableExpr(result, n.Right, defaultDatabase)
switch condition := n.Condition.(type) {
case *ast.OnCondition:
- collectOmniAccessTablesFromExpr(result, condition.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, condition.Expr, defaultDatabase)
default:
}
case *ast.SubqueryExpr:
- collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase)
case *ast.JsonTableExpr:
- collectOmniAccessTablesFromExpr(result, n.Expr, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, n.Path, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, n.Expr, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, n.Path, defaultDatabase)
default:
}
}
-func collectOmniAccessTablesFromExpr(result base.SourceColumnSet, expr ast.ExprNode, defaultDatabase string, normalizeStarRocksCluster bool) {
+func collectOmniAccessTablesFromExpr(result base.SourceColumnSet, expr ast.ExprNode, defaultDatabase string) {
switch e := expr.(type) {
case nil:
case *ast.ResTarget:
- collectOmniAccessTablesFromExpr(result, e.Val, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Val, defaultDatabase)
case *ast.BinaryExpr:
- collectOmniAccessTablesFromExpr(result, e.Left, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, e.Right, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Left, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, e.Right, defaultDatabase)
case *ast.UnaryExpr:
- collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase)
case *ast.FuncCallExpr:
for _, arg := range e.Args {
- collectOmniAccessTablesFromExpr(result, arg, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, arg, defaultDatabase)
}
- collectOmniAccessTablesFromExpr(result, e.Separator, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Separator, defaultDatabase)
for _, orderBy := range e.OrderBy {
if orderBy != nil {
- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)
}
}
if e.Over != nil {
for _, partitionBy := range e.Over.PartitionBy {
- collectOmniAccessTablesFromExpr(result, partitionBy, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, partitionBy, defaultDatabase)
}
for _, orderBy := range e.Over.OrderBy {
if orderBy != nil {
- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)
}
}
}
case *ast.SubqueryExpr:
- collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase)
case *ast.CaseExpr:
- collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase)
for _, when := range e.Whens {
if when == nil {
continue
}
- collectOmniAccessTablesFromExpr(result, when.Cond, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, when.Result, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, when.Cond, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, when.Result, defaultDatabase)
}
- collectOmniAccessTablesFromExpr(result, e.Default, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Default, defaultDatabase)
case *ast.BetweenExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, e.Low, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, e.High, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, e.Low, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, e.High, defaultDatabase)
case *ast.InExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
for _, item := range e.List {
- collectOmniAccessTablesFromExpr(result, item, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, item, defaultDatabase)
}
- collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase)
case *ast.LikeExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, e.Pattern, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, e.Escape, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, e.Pattern, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, e.Escape, defaultDatabase)
case *ast.IsExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
case *ast.ExistsExpr:
- collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase)
case *ast.CastExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
case *ast.ExtractExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
case *ast.IntervalExpr:
- collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase)
case *ast.CollateExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
case *ast.MatchExpr:
for _, column := range e.Columns {
- collectOmniAccessTablesFromExpr(result, column, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, column, defaultDatabase)
}
- collectOmniAccessTablesFromExpr(result, e.Against, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Against, defaultDatabase)
case *ast.ConvertExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
case *ast.RowExpr:
for _, item := range e.Items {
- collectOmniAccessTablesFromExpr(result, item, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, item, defaultDatabase)
}
case *ast.MemberOfExpr:
- collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase, normalizeStarRocksCluster)
- collectOmniAccessTablesFromExpr(result, e.Array, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase)
+ collectOmniAccessTablesFromExpr(result, e.Array, defaultDatabase)
case *ast.ParenExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
case *ast.WeightStringExpr:
- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)
+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)
default:
}
}
-
-func filterStarRocksClusterName(databaseName string) string {
- list := strings.Split(databaseName, ":")
- if len(list) > 1 {
- return list[len(list)-1]
- }
- return databaseName
-}
diff --git a/backend/plugin/parser/mysql/query_span_extractor_test.go b/backend/plugin/parser/mysql/query_span_extractor_test.go
index 01f25204cd..198db56183 100644
--- a/backend/plugin/parser/mysql/query_span_extractor_test.go
+++ b/backend/plugin/parser/mysql/query_span_extractor_test.go
@@ -1393,46 +1393,7 @@ func TestOmniQuerySpanScenarioBatch4_CaseSensitivityCoverage(t *testing.T) {
})
}
-func TestOmniQuerySpanScenarioBatch5_StarRocksAndMetadataCoverage(t *testing.T) {
- t.Run("starrocks_cluster_qualified_tables", func(t *testing.T) {
- gCtx := newOmniStarRocksTestQuerySpanContext()
- span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(
- context.Background(),
- "SELECT `cluster_name:db`.products.product_info FROM `cluster_name:db`.products",
- )
- require.NoError(t, err)
- require.Equal(t, base.Select, span.Type)
- require.Equal(t, []base.QuerySpanResult{
- {Name: "product_info", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},
- }, span.Results)
- require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products"}}), span.SourceColumns)
- })
-
- t.Run("starrocks_system_user_mixing_uses_normalized_database", func(t *testing.T) {
- gCtx := newOmniStarRocksTestQuerySpanContext()
- _, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(
- context.Background(),
- "SELECT * FROM products, `cluster_name:information_schema`.tables",
- )
- require.ErrorIs(t, err, base.MixUserSystemTablesError)
- })
-
- t.Run("mysql_and_starrocks_lineage_match_without_cluster_prefix", func(t *testing.T) {
- mysqlSpan, err := newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), false).getOmniQuerySpan(
- context.Background(),
- "SELECT product_info FROM products",
- )
- require.NoError(t, err)
-
- starrocksSpan, err := newOmniQuerySpanExtractor("db", newOmniStarRocksTestQuerySpanContext(), false).getOmniQuerySpan(
- context.Background(),
- "SELECT product_info FROM products",
- )
- require.NoError(t, err)
- require.Equal(t, mysqlSpan.Results, starrocksSpan.Results)
- require.Equal(t, mysqlSpan.SourceColumns, starrocksSpan.SourceColumns)
- })
-
+func TestOmniQuerySpanScenarioBatch5_MetadataCoverage(t *testing.T) {
t.Run("duplicate_database_case_uses_ignore_case_setting", func(t *testing.T) {
gCtx := newOmniCaseCollisionTestQuerySpanContext()
@@ -1570,12 +1531,6 @@ func newOmniTestQuerySpanContext() base.GetQuerySpanContext {
}
}
-func newOmniStarRocksTestQuerySpanContext() base.GetQuerySpanContext {
- gCtx := newOmniTestQuerySpanContext()
- gCtx.Engine = storepb.Engine_STARROCKS
- return gCtx
-}
-
func newOmniCaseCollisionTestQuerySpanContext() base.GetQuerySpanContext {
dbMetadata := &metadatapb.DatabaseSchemaMetadata{
Name: "db",
diff --git a/backend/plugin/parser/mysql/query_span_parity_test.go b/backend/plugin/parser/mysql/query_span_parity_test.go
index 150ed77575..3d9938699b 100644
--- a/backend/plugin/parser/mysql/query_span_parity_test.go
+++ b/backend/plugin/parser/mysql/query_span_parity_test.go
@@ -43,9 +43,6 @@ func TestMySQLOmniQuerySpanGoldenHarness(t *testing.T) {
total := 0
for _, testDataPath := range mysqlOmniProbeFixturePaths {
engine := storepb.Engine_MYSQL
- if strings.Contains(testDataPath, "starrocks") {
- engine = storepb.Engine_STARROCKS
- }
yamlFile, err := os.Open(testDataPath)
if err != nil {
diff --git a/backend/plugin/parser/mysql/query_span_probe_test.go b/backend/plugin/parser/mysql/query_span_probe_test.go
index a95c34e2fa..387a944d2b 100644
--- a/backend/plugin/parser/mysql/query_span_probe_test.go
+++ b/backend/plugin/parser/mysql/query_span_probe_test.go
@@ -30,7 +30,6 @@ var mysqlOmniProbeFixturePaths = []string{
"test-data/query-span/query_type.yaml",
"test-data/query-span/standard.yaml",
"test-data/query-span/case_insensitive.yaml",
- "test-data/query-span/starrocks.yaml",
}
func probeMySQLOmniFixtureParseCoverage(t *testing.T) {
diff --git a/backend/plugin/parser/mysql/query_span_test.go b/backend/plugin/parser/mysql/query_span_test.go
index 74b9d68d65..81034c93ff 100644
--- a/backend/plugin/parser/mysql/query_span_test.go
+++ b/backend/plugin/parser/mysql/query_span_test.go
@@ -4,7 +4,6 @@ import (
"context"
"io"
"os"
- "strings"
"testing"
metadatapb "github.com/bytebase/omni/metadata"
@@ -37,16 +36,12 @@ func TestGetQuerySpan(t *testing.T) {
"test-data/query-span/query_type.yaml",
"test-data/query-span/standard.yaml",
"test-data/query-span/case_insensitive.yaml",
- "test-data/query-span/starrocks.yaml",
}
)
a := require.New(t)
for _, testDataPath := range testDataPaths {
engine := storepb.Engine_MYSQL
- if strings.Contains(testDataPath, "starrocks") {
- engine = storepb.Engine_STARROCKS
- }
testDataPath := testDataPath
@@ -175,7 +170,7 @@ func TestExtractTableRefs(t *testing.T) {
require.NoError(t, err, "failed to parse statement: %s", test.statement)
require.Len(t, parseResult.Items, 1, "expected one parse result for statement: %s", test.statement)
- sourceColumns := collectOmniAccessTables(parseResult.Items[0], "db", false)
+ sourceColumns := collectOmniAccessTables(parseResult.Items[0], "db")
var resources []base.SchemaResource
for resource := range sourceColumns {
resources = append(resources, base.SchemaResource{
diff --git a/backend/plugin/parser/mysql/resource_change.go b/backend/plugin/parser/mysql/resource_change.go
index 0109f58852..852c48d36b 100644
--- a/backend/plugin/parser/mysql/resource_change.go
+++ b/backend/plugin/parser/mysql/resource_change.go
@@ -15,8 +15,6 @@ import (
func init() {
base.RegisterExtractChangedResourcesFunc(storepb.Engine_MYSQL, extractChangedResources)
base.RegisterExtractChangedResourcesFunc(storepb.Engine_MARIADB, extractChangedResources)
- base.RegisterExtractChangedResourcesFunc(storepb.Engine_STARROCKS, extractChangedResources)
- base.RegisterExtractChangedResourcesFunc(storepb.Engine_DORIS, extractChangedResources)
}
func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {
diff --git a/backend/plugin/parser/mysql/test-data/query-span/starrocks.yaml b/backend/plugin/parser/mysql/test-data/query-span/starrocks.yaml
deleted file mode 100644
index 2fb7bfe85a..0000000000
--- a/backend/plugin/parser/mysql/test-data/query-span/starrocks.yaml
+++ /dev/null
@@ -1,100 +0,0 @@
-- description: Test For cluster name
- statement: SELECT * FROM v2
- defaultDatabase: db
- metadata: |-
- {
- "name": "db",
- "schemas": [
- {
- "name": "",
- "tables": [
- {
- "name": "products",
- "columns": [
- {
- "name": "product_info"
- }
- ]
- }
- ],
- "views": [
- {
- "name": "v1",
- "definition": "SELECT `cluster_name:db`.`products`.`product_info` FROM `cluster_name:db`.`products`"
- },
- {
- "name": "v2",
- "definition": "SELECT `cluster_name:db`.`v1`.`product_info` FROM `cluster_name:db`.`v1`"
- }
- ]
- }
- ]
- }
- querySpan:
- type: 1
- results:
- - name: product_info
- sourcecolumns:
- - server: ""
- database: db
- schema: ""
- table: products
- column: product_info
- isplainfield: true
- sourcefieldpaths: []
- selectasterisk: false
- sourcecolumns:
- - server: ""
- database: db
- schema: ""
- table: v2
- column: ""
- predicatecolumns: []
-- description: Test For cluster name
- statement: SELECT * FROM v1
- defaultDatabase: db
- metadata: |-
- {
- "name": "db",
- "schemas": [
- {
- "name": "",
- "tables": [
- {
- "name": "products",
- "columns": [
- {
- "name": "product_info"
- }
- ]
- }
- ],
- "views": [
- {
- "name": "v1",
- "definition": "SELECT `cluster_name:db`.`products`.`product_info` FROM `cluster_name:db`.`products`"
- }
- ]
- }
- ]
- }
- querySpan:
- type: 1
- results:
- - name: product_info
- sourcecolumns:
- - server: ""
- database: db
- schema: ""
- table: products
- column: product_info
- isplainfield: true
- sourcefieldpaths: []
- selectasterisk: false
- sourcecolumns:
- - server: ""
- database: db
- schema: ""
- table: v1
- column: ""
- predicatecolumns: []
diff --git a/backend/plugin/parser/starrocks/resource_change.go b/backend/plugin/parser/starrocks/resource_change.go
new file mode 100644
index 0000000000..728ebfc6dd
--- /dev/null
+++ b/backend/plugin/parser/starrocks/resource_change.go
@@ -0,0 +1,163 @@
+package starrocks
+
+import (
+ "strings"
+
+ "github.com/bytebase/omni/starrocks/ast"
+ "github.com/pkg/errors"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store/model"
+)
+
+func init() {
+ base.RegisterExtractChangedResourcesFunc(storepb.Engine_STARROCKS, extractChangedResources)
+}
+
+func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {
+ changedResources := model.NewChangedResources(dbMetadata)
+
+ var dmlCount, insertCount int
+ var dmlStatements []string
+
+ addTable := func(name *ast.ObjectName, affectsExistingRows bool) {
+ db, table := objectName(name, currentDatabase)
+ if table == "" {
+ return
+ }
+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, affectsExistingRows)
+ }
+
+ // addObjectDatabase records a database-only write target for a non-table object DDL
+ // (view, materialized view, routine) ONLY when the object's own name carries an
+ // explicit database qualifier. An unqualified name records nothing, leaving the
+ // request database to stand in. SUP-222.
+ addObjectDatabase := func(name *ast.ObjectName) {
+ if db, _ := objectName(name, ""); db != "" {
+ changedResources.AddDatabase(db)
+ }
+ }
+
+ for _, unifiedAST := range asts {
+ parsed, ok := unifiedAST.(*omniAST)
+ if !ok {
+ return nil, errors.New("expected omniAST for StarRocks")
+ }
+ if parsed.node == nil {
+ continue
+ }
+
+ switch n := parsed.node.(type) {
+ case *ast.CreateTableStmt:
+ addTable(n.Name, false)
+
+ case *ast.DropTableStmt:
+ addTable(n.Name, true)
+
+ case *ast.TruncateTableStmt:
+ addTable(n.Target, true)
+
+ case *ast.AlterTableStmt:
+ addTable(n.Name, true)
+
+ case *ast.CreateIndexStmt:
+ addTable(n.Table, false)
+
+ case *ast.DropIndexStmt:
+ addTable(n.Table, false)
+
+ case *ast.LoadDataStmt:
+ // One LOAD LABEL carries a data description per target table.
+ for _, desc := range n.DataDescs {
+ if desc != nil {
+ addTable(desc.Target, false)
+ }
+ }
+
+ case *ast.InsertStmt:
+ // INSERT INTO FILES(...) writes to external storage and leaves Target nil,
+ // so it records no table while still counting as a DML statement.
+ addTable(n.Target, false)
+ if len(n.Values) > 0 {
+ insertCount += len(n.Values)
+ continue
+ }
+ dmlCount++
+ dmlStatements = append(dmlStatements, statementText(parsed))
+
+ case *ast.UpdateStmt:
+ // StarRocks updates a single target; FROM names read sources, not write targets.
+ addTable(n.Target, false)
+ dmlCount++
+ dmlStatements = append(dmlStatements, statementText(parsed))
+
+ case *ast.DeleteStmt:
+ // USING names read sources, as FROM does for UPDATE.
+ addTable(n.Target, false)
+ dmlCount++
+ dmlStatements = append(dmlStatements, statementText(parsed))
+
+ case *ast.MergeStmt:
+ addTable(n.Target, false)
+ dmlCount++
+ dmlStatements = append(dmlStatements, statementText(parsed))
+
+ // Non-table object DDL: gate by the object's own explicit database qualifier only.
+ // CREATE FUNCTION is absent because its AST keeps the name inside RawSignature
+ // rather than as an ObjectName, so there is nothing to qualify by.
+ case *ast.CreateViewStmt:
+ addObjectDatabase(n.Name)
+ case *ast.AlterViewStmt:
+ addObjectDatabase(n.Name)
+ case *ast.DropViewStmt:
+ addObjectDatabase(n.Name)
+ case *ast.CreateMTMVStmt:
+ addObjectDatabase(n.Name)
+ case *ast.AlterMTMVStmt:
+ addObjectDatabase(n.Name)
+ case *ast.DropMTMVStmt:
+ addObjectDatabase(n.Name)
+ case *ast.DropFunctionStmt:
+ addObjectDatabase(n.Name)
+ case *ast.CreateProcedureStmt:
+ addObjectDatabase(n.Name)
+ case *ast.DropProcedureStmt:
+ addObjectDatabase(n.Name)
+
+ default:
+ }
+ }
+
+ return &base.ChangeSummary{
+ ChangedResources: changedResources,
+ DMLStatements: dmlStatements,
+ DMLCount: dmlCount,
+ InsertCount: insertCount,
+ }, nil
+}
+
+// objectName splits an ObjectName into its database and object name. StarRocks writes a
+// name as table, db.table or catalog.db.table, so the object is the last part and the
+// database the one before it; a catalog qualifier is dropped, since changed resources
+// are tracked per database.
+func objectName(name *ast.ObjectName, defaultDatabase string) (string, string) {
+ if name == nil || len(name.Parts) == 0 {
+ return defaultDatabase, ""
+ }
+ parts := name.Parts
+ database := defaultDatabase
+ if len(parts) >= 2 {
+ database = parts[len(parts)-2]
+ }
+ return database, parts[len(parts)-1]
+}
+
+// statementText returns the text of a statement, ensuring it ends with a semicolon.
+func statementText(a *omniAST) string {
+ text := strings.TrimSpace(a.text)
+ if !strings.HasSuffix(text, ";") {
+ text += ";"
+ }
+ return text
+}
diff --git a/backend/plugin/parser/starrocks/resource_change_test.go b/backend/plugin/parser/starrocks/resource_change_test.go
new file mode 100644
index 0000000000..31b71148e1
--- /dev/null
+++ b/backend/plugin/parser/starrocks/resource_change_test.go
@@ -0,0 +1,149 @@
+package starrocks
+
+import (
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ metadatapb "github.com/bytebase/omni/metadata"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+ "github.com/bytebase/bytebase/backend/plugin/parser/base"
+ "github.com/bytebase/bytebase/backend/store/model"
+)
+
+// changedTables flattens the built resources into "database.table" strings.
+func changedTables(t *testing.T, statement string) []string {
+ t.Helper()
+ summary := extract(t, statement, nil)
+ var result []string
+ for _, database := range summary.ChangedResources.Build().GetDatabases() {
+ for _, schema := range database.GetSchemas() {
+ for _, table := range schema.GetTables() {
+ result = append(result, database.GetName()+"."+table.GetName())
+ }
+ }
+ }
+ return result
+}
+
+func extract(t *testing.T, statement string, dbMetadata *model.DatabaseMetadata) *base.ChangeSummary {
+ t.Helper()
+ stmts, err := base.ParseStatements(storepb.Engine_STARROCKS, statement)
+ require.NoError(t, err)
+ summary, err := base.ExtractChangedResources(storepb.Engine_STARROCKS, "db", "", dbMetadata, base.ExtractASTs(stmts), statement)
+ require.NoError(t, err)
+ return summary
+}
+
+func TestExtractChangedResourcesTables(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ want []string
+ }{
+ {"create table takes the request database", "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"db.t"}},
+ {"a qualified name keeps its own database", "CREATE TABLE other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"other.t"}},
+ {"a catalog qualifier is dropped", "CREATE TABLE cat.other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"other.t"}},
+ {"drop table", "DROP TABLE t;", []string{"db.t"}},
+ {"truncate table", "TRUNCATE TABLE t;", []string{"db.t"}},
+ {"alter table", "ALTER TABLE t ADD COLUMN b INT;", []string{"db.t"}},
+ {"create index names its table", "CREATE INDEX idx ON t (a);", []string{"db.t"}},
+ {"drop index names its table", "DROP INDEX idx ON t;", []string{"db.t"}},
+ {"insert names its target", "INSERT INTO t VALUES (1);", []string{"db.t"}},
+ {"update names its target, not its source", "UPDATE t SET a = 1 WHERE a IN (SELECT a FROM s);", []string{"db.t"}},
+ {"delete names its target, not its source", "DELETE FROM t WHERE a = 1;", []string{"db.t"}},
+ {"a view records no table", "CREATE VIEW v AS SELECT 1;", nil},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ require.ElementsMatch(t, tc.want, changedTables(t, tc.statement))
+ })
+ }
+}
+
+func TestExtractChangedResourcesDMLAccounting(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ wantDML int
+ wantInserts int
+ wantTexts []string
+ }{
+ {name: "insert values counts rows, not statements", statement: "INSERT INTO t VALUES (1), (2), (3);", wantInserts: 3},
+ {name: "insert select is a DML statement", statement: "INSERT INTO t SELECT a FROM s;", wantDML: 1, wantTexts: []string{"INSERT INTO t SELECT a FROM s;"}},
+ {name: "update is a DML statement", statement: "UPDATE t SET a = 1 WHERE a = 2;", wantDML: 1, wantTexts: []string{"UPDATE t SET a = 1 WHERE a = 2;"}},
+ {name: "delete is a DML statement", statement: "DELETE FROM t WHERE a = 1;", wantDML: 1, wantTexts: []string{"DELETE FROM t WHERE a = 1;"}},
+ {name: "DDL is neither", statement: "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;"},
+ {
+ name: "a script accumulates both",
+ statement: "INSERT INTO t VALUES (1), (2);\nUPDATE t SET a = 1;\nDELETE FROM t;",
+ wantDML: 2,
+ wantInserts: 2,
+ wantTexts: []string{"UPDATE t SET a = 1;", "DELETE FROM t;"},
+ },
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ summary := extract(t, tc.statement, nil)
+ require.Equal(t, tc.wantDML, summary.DMLCount)
+ require.Equal(t, tc.wantInserts, summary.InsertCount)
+ require.Equal(t, tc.wantTexts, summary.DMLStatements)
+ })
+ }
+}
+
+func TestExtractChangedResourcesDatabaseOnlyTargets(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ statement string
+ want []string
+ }{
+ {"a qualified view records its database", "CREATE VIEW other.v AS SELECT 1;", []string{"other"}},
+ {"an unqualified view records nothing", "CREATE VIEW v AS SELECT 1;", []string{}},
+ {"alter view", "ALTER VIEW other.v AS SELECT 1;", []string{"other"}},
+ {"drop view", "DROP VIEW other.v;", []string{"other"}},
+ {"a qualified materialized view records its database", "DROP MATERIALIZED VIEW other.mv;", []string{"other"}},
+ {"drop procedure", "DROP PROCEDURE other.p;", []string{"other"}},
+ {"a table DDL records no database-only target", "CREATE TABLE other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ summary := extract(t, tc.statement, nil)
+ require.Equal(t, tc.want, summary.ChangedResources.GetDatabaseOnlyTargets())
+ })
+ }
+}
+
+func TestExtractChangedResourcesAffectedRows(t *testing.T) {
+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{
+ Name: "db",
+ Schemas: []*metadatapb.SchemaMetadata{{Tables: []*metadatapb.TableMetadata{{Name: "t", RowCount: 42}}}},
+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_STARROCKS, false /* caseSensitive */)
+
+ for _, tc := range []struct {
+ name string
+ statement string
+ want int64
+ }{
+ {"drop discards existing rows", "DROP TABLE t;", 42},
+ {"truncate discards existing rows", "TRUNCATE TABLE t;", 42},
+ {"alter rewrites existing rows", "ALTER TABLE t ADD COLUMN b INT;", 42},
+ {"create has no existing rows", "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", 0},
+ {"insert leaves existing rows alone", "INSERT INTO t VALUES (1);", 0},
+ {"create index leaves existing rows alone", "CREATE INDEX idx ON t (a);", 0},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ summary := extract(t, tc.statement, dbMetadata)
+ require.Equal(t, tc.want, summary.ChangedResources.CountAffectedTableRows())
+ })
+ }
+}
+
+// foreignAST stands in for another engine's AST, which reached this extractor
+// through the shared registry before StarRocks had one of its own.
+type foreignAST struct{}
+
+func (*foreignAST) ASTStartPosition() *storepb.Position { return nil }
+
+func TestExtractChangedResourcesRejectsForeignAST(t *testing.T) {
+ _, err := extractChangedResources("db", "", nil, []base.AST{&foreignAST{}}, "")
+ require.ErrorContains(t, err, "expected omniAST for StarRocks")
+}
diff --git a/backend/plugin/parser/starrocks/starrocks.go b/backend/plugin/parser/starrocks/starrocks.go
index c14dc889eb..0a1bef9860 100644
--- a/backend/plugin/parser/starrocks/starrocks.go
+++ b/backend/plugin/parser/starrocks/starrocks.go
@@ -19,6 +19,7 @@ func init() {
// omniAST wraps an omni AST node to implement the base.AST interface.
type omniAST struct {
node ast.Node
+ text string
startPos *storepb.Position
}
@@ -67,6 +68,7 @@ func parseStarRocksStatements(statement string) ([]base.ParsedStatement, error)
}
ps.AST = &omniAST{
node: node,
+ text: stmt.Text,
startPos: stmt.Start,
}
result = append(result, ps)
@@ -103,6 +105,7 @@ func parseStarRocksSQL(statement string) ([]*omniAST, error) {
}
result = append(result, &omniAST{
node: node,
+ text: stmt.Text,
startPos: stmt.Start,
})
}
diff --git a/backend/server/ultimate.go b/backend/server/ultimate.go
index 139f81a2ee..5f1390dd21 100644
--- a/backend/server/ultimate.go
+++ b/backend/server/ultimate.go
@@ -8,6 +8,7 @@ import (
_ "github.com/bytebase/bytebase/backend/plugin/db/cockroachdb"
_ "github.com/bytebase/bytebase/backend/plugin/db/cosmosdb"
_ "github.com/bytebase/bytebase/backend/plugin/db/databricks"
+ _ "github.com/bytebase/bytebase/backend/plugin/db/doris"
_ "github.com/bytebase/bytebase/backend/plugin/db/dynamodb"
_ "github.com/bytebase/bytebase/backend/plugin/db/elasticsearch"
_ "github.com/bytebase/bytebase/backend/plugin/db/hive"