Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
refactor(doris,starrocks): split the engines, and fix their changed-resource extraction (#21731)
Danny Xu07c8fc2a8b68
diff --git a/backend/plugin/db/doris/doris.go b/backend/plugin/db/doris/doris.gonew file mode 100644index 0000000000..52449f7a3e--- /dev/null+++ b/backend/plugin/db/doris/doris.go@@ -0,0 +1,398 @@+// Package doris is the plugin for the Apache Doris driver.+package doris++import (+ "context"+ "database/sql"+ "fmt"+ "log/slog"+ "net"+ "regexp"+ "strings"+ "time"++ dorisomni "github.com/bytebase/omni/doris/parser"+ "github.com/go-sql-driver/mysql"+ "github.com/google/uuid"+ "github.com/pkg/errors"+ "go.uber.org/multierr"+ "golang.org/x/crypto/ssh"+ "google.golang.org/protobuf/types/known/durationpb"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/db"+ "github.com/bytebase/bytebase/backend/plugin/db/transaction"+ "github.com/bytebase/bytebase/backend/plugin/db/util"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"++ // Register how Doris plans a statement, for base.ExplainStatement below.+ _ "github.com/bytebase/bytebase/backend/plugin/parser/doris"+)++var (+ baseTableType = "BASE TABLE"+ viewTableType = "VIEW"+ materializedViewType = "MATERIALIZED VIEW"++ _ db.Driver = (*Driver)(nil)+)++func init() {+ db.Register(storepb.Engine_DORIS, newDriver)+}++// Driver is the Doris driver.+type Driver struct {+ dbType storepb.Engine+ db *sql.DB+ databaseName string+ sshClient *ssh.Client++ // Called upon driver.Open() finishes.+ openCleanUp []func()+}++func newDriver() db.Driver {+ return &Driver{}+}++// Open opens a Doris driver.+func (d *Driver) Open(_ context.Context, dbType storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {+ defer func() {+ for _, f := range d.openCleanUp {+ f()+ }+ }()++ protocol := "tcp"+ if strings.HasPrefix(connCfg.DataSource.Host, "/") {+ protocol = "unix"+ }+ params := []string{"multiStatements=true", "maxAllowedPacket=0", "interpolateParams=true"}+ if connCfg.DataSource.GetSshHost() != "" {+ sshClient, err := util.GetSSHClient(connCfg.DataSource)+ if err != nil {+ return nil, err+ }+ d.sshClient = sshClient+ // Now we register the dialer with the ssh connection as a parameter.+ protocol = "mysql-tcp-" + uuid.NewString()[:8]+ // Now we register the dialer with the ssh connection as a parameter.+ mysql.RegisterDialContext(protocol, func(_ context.Context, addr string) (net.Conn, error) {+ return sshClient.Dial("tcp", addr)+ })+ }++ tlscfg, err := util.GetTLSConfig(connCfg.DataSource)+ if err != nil {+ return nil, errors.Wrap(err, "sql: tls config error")+ }+ tlsKey := uuid.NewString()+ if tlscfg != nil {+ if err := mysql.RegisterTLSConfig(tlsKey, tlscfg); err != nil {+ return nil, errors.Wrap(err, "sql: failed to register tls config")+ }+ // TLS config is only used during sql.Open, so should be safe to deregister afterwards.+ d.openCleanUp = append(d.openCleanUp, func() { mysql.DeregisterTLSConfig(tlsKey) })+ params = append(params, fmt.Sprintf("tls=%s", tlsKey))+ }++ dsn := fmt.Sprintf("%s:%s@%s(%s:%s)/%s?%s", connCfg.DataSource.Username, connCfg.Password, protocol, connCfg.DataSource.Host, connCfg.DataSource.Port, connCfg.ConnectionContext.DatabaseName, strings.Join(params, "&"))+ db, err := sql.Open("mysql", dsn)+ if err != nil {+ return nil, err+ }+ d.dbType = dbType+ d.db = db+ // TODO(d): remove the work-around once we have clean-up the migration connection hack.+ db.SetConnMaxLifetime(2 * time.Hour)+ db.SetMaxOpenConns(50)+ db.SetMaxIdleConns(15)+ d.databaseName = connCfg.ConnectionContext.DatabaseName+ return d, nil+}++// Close closes the driver.+func (d *Driver) Close(context.Context) error {+ var err error+ err = multierr.Append(err, d.db.Close())+ if d.sshClient != nil {+ err = multierr.Append(err, d.sshClient.Close())+ }+ return err+}++// Ping pings the database.+func (d *Driver) Ping(ctx context.Context) error {+ return d.db.PingContext(ctx)+}++// GetDB gets the database.+func (d *Driver) GetDB() *sql.DB {+ return d.db+}++// getVersion gets the version.+func (d *Driver) getVersion(ctx context.Context) (string, string, error) {+ query := "SELECT VERSION()"+ var version string+ if err := d.db.QueryRowContext(ctx, query).Scan(&version); err != nil {+ if err == sql.ErrNoRows {+ return "", "", common.FormatDBErrorEmptyRowWithQuery(query)+ }+ return "", "", util.FormatErrorWithQuery(err, query)+ }++ return parseVersion(version)+}++func parseVersion(version string) (string, string, error) {+ if loc := regexp.MustCompile(`^\d+.\d+.\d+`).FindStringIndex(version); loc != nil {+ return version[loc[0]:loc[1]], version[loc[1]:], nil+ }+ return "", "", errors.Errorf("failed to parse version %q", version)+}++// Execute executes a SQL statement.+func (d *Driver) Execute(ctx context.Context, statement string, opts db.ExecuteOptions) (int64, error) {+ // Parse transaction mode from the script+ config, cleanedStatement := base.ParseTransactionConfig(statement)+ statement = cleanedStatement+ transactionMode := config.Mode++ // Apply default when transaction mode is not specified+ if transactionMode == transaction.ModeUnspecified {+ transactionMode = transaction.DefaultMode()+ }++ // Only preprocess when DELIMITER directives are present; normal+ // Doris SQL passes through unchanged.+ if containsDelimiterDirective(statement) {+ segments := dorisomni.Split(statement)+ var buf strings.Builder+ for i, seg := range segments {+ if i > 0 {+ buf.WriteString(";\n")+ }+ buf.WriteString(seg.Text)+ }+ statement = buf.String()+ }+ // Execute based on transaction mode+ // Note: Doris is an OLAP database with limited transaction support.+ // For DDL operations, transactions are not supported. For DML operations,+ // Doris supports transactions within certain limitations.+ if transactionMode == transaction.ModeOff || opts.CreateDatabase {+ return d.executeInAutoCommitMode(ctx, statement)+ }+ return d.executeInTransactionMode(ctx, statement, opts)+}++func (d *Driver) executeInTransactionMode(ctx context.Context, statement string, opts db.ExecuteOptions) (int64, error) {+ conn, err := d.db.Conn(ctx)+ if err != nil {+ return 0, err+ }+ defer conn.Close()++ connectionID, err := getConnectionID(ctx, conn)+ if err != nil {+ return 0, err+ }+ slog.Debug("connectionID", slog.String("connectionID", connectionID))++ tx, err := conn.BeginTx(ctx, nil)+ if err != nil {+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_BEGIN, err.Error())+ return 0, errors.Wrapf(err, "failed to begin execute transaction")+ }+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_BEGIN, "")++ committed := false+ defer func() {+ if !committed {+ err := tx.Rollback()+ var rerr string+ if err != nil && !errors.Is(err, sql.ErrTxDone) {+ rerr = err.Error()+ slog.Debug("failed to rollback transaction", log.BBError(err))+ }+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_ROLLBACK, rerr)+ }+ }()++ sqlResult, err := tx.ExecContext(ctx, statement)+ if err != nil {+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {+ slog.Info("cancel connection", slog.String("connectionID", connectionID))+ if err := d.StopConnectionByID(connectionID); err != nil {+ slog.Error("failed to cancel connection", slog.String("connectionID", connectionID), log.BBError(err))+ }+ }+ return 0, err+ }++ rowsAffected, err := sqlResult.RowsAffected()+ if err != nil {+ // Since we cannot differentiate DDL and DML yet, we have to ignore the error.+ slog.Debug("rowsAffected returns error", log.BBError(err))+ }++ if err := tx.Commit(); err != nil {+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_COMMIT, err.Error())+ return 0, errors.Wrapf(err, "failed to commit execute transaction")+ }+ opts.LogTransactionControl(storepb.TaskRunLog_TransactionControl_COMMIT, "")+ committed = true+ return rowsAffected, nil+}++func (d *Driver) executeInAutoCommitMode(ctx context.Context, statement string) (int64, error) {+ conn, err := d.db.Conn(ctx)+ if err != nil {+ return 0, err+ }+ defer conn.Close()++ connectionID, err := getConnectionID(ctx, conn)+ if err != nil {+ return 0, err+ }+ slog.Debug("connectionID", slog.String("connectionID", connectionID))++ // Execute in auto-commit mode+ sqlResult, err := conn.ExecContext(ctx, statement)+ if err != nil {+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {+ slog.Info("cancel connection", slog.String("connectionID", connectionID))+ if err := d.StopConnectionByID(connectionID); err != nil {+ slog.Error("failed to cancel connection", slog.String("connectionID", connectionID), log.BBError(err))+ }+ }+ return 0, err+ }++ rowsAffected, err := sqlResult.RowsAffected()+ if err != nil {+ // Since we cannot differentiate DDL and DML yet, we have to ignore the error.+ slog.Debug("rowsAffected returns error", log.BBError(err))+ }+ return rowsAffected, nil+}++// QueryConn queries a SQL statement in a given connection.+func (d *Driver) QueryConn(ctx context.Context, conn *sql.Conn, statement string, queryContext db.QueryContext) ([]*v1pb.QueryResult, error) {+ singleSQLs, err := base.SplitMultiSQL(d.dbType, statement)+ if err != nil {+ return nil, err+ }+ singleSQLs = base.FilterEmptyStatements(singleSQLs)+ if len(singleSQLs) == 0 {+ return nil, nil+ }++ connectionID, err := getConnectionID(ctx, conn)+ if err != nil {+ return nil, err+ }+ slog.Debug("connectionID", slog.String("connectionID", connectionID))++ var results []*v1pb.QueryResult+ for _, singleSQL := range singleSQLs {+ statement := singleSQL.Text+ if queryContext.Explain {+ explained, err := base.ExplainStatement(d.dbType, statement, db.ExplainFormat(queryContext.Option.GetExplainFormat()))+ if err != nil {+ return nil, err+ }+ statement = explained+ } else if queryContext.Limit > 0 {+ statement = base.StatementWithResultLimit(d.dbType, statement, queryContext.Limit, "")+ }+ sqlWithBytebaseAppComment := util.MySQLPrependBytebaseAppComment(statement)++ _, allQuery, err := base.ValidateSQLForEditor(d.dbType, statement)+ if err != nil {+ // TODO(d): need to make parser compatible.+ slog.Error("failed to validate sql", slog.String("statement", statement), log.BBError(err))+ allQuery = true+ }+ startTime := time.Now()+ queryResult, err := func() (*v1pb.QueryResult, error) {+ if allQuery {+ rows, err := conn.QueryContext(ctx, sqlWithBytebaseAppComment)+ if err != nil {+ return nil, err+ }+ defer rows.Close()+ r, err := util.RowsToQueryResult(rows, makeValueByTypeName, convertValue, queryContext.MaximumSQLResultSize)+ if err != nil {+ return nil, err+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return r, nil+ }++ sqlResult, err := conn.ExecContext(ctx, statement)+ if err != nil {+ return nil, err+ }+ affectedRows, err := sqlResult.RowsAffected()+ if err != nil {+ slog.Info("rowsAffected returns error", log.BBError(err))+ }+ return util.BuildAffectedRowsResult(affectedRows, nil), nil+ }()+ stop := false+ if err != nil {+ if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {+ slog.Info("cancel connection", slog.String("connectionID", connectionID))+ if err := d.StopConnectionByID(connectionID); err != nil {+ slog.Error("failed to cancel connection", slog.String("connectionID", connectionID), log.BBError(err))+ }+ }+ queryResult = &v1pb.QueryResult{+ Error: err.Error(),+ }+ stop = true+ }+ queryResult.Statement = statement+ queryResult.Latency = durationpb.New(time.Since(startTime))+ queryResult.RowsCount = int64(len(queryResult.Rows))+ results = append(results, queryResult)+ if stop {+ break+ }+ }++ return results, nil+}++func (d *Driver) StopConnectionByID(id string) error {+ _, err := d.db.Exec(fmt.Sprintf("KILL QUERY %s", id)) // NOSONAR(go:S2077) id is from CONNECTION_ID() server function, not user input+ return err+}++func containsDelimiterDirective(s string) bool {+ for _, line := range strings.Split(s, "\n") {+ trimmed := strings.TrimLeft(line, " \t")+ upper := strings.ToUpper(trimmed)+ if strings.HasPrefix(upper, "DELIMITER ") || strings.HasPrefix(upper, "DELIMITER\t") {+ return true+ }+ }+ return false+}++func getConnectionID(ctx context.Context, conn *sql.Conn) (string, error) {+ var id string+ if err := conn.QueryRowContext(ctx, `SELECT CONNECTION_ID();`).Scan(&id); err != nil {+ return "", err+ }+ return id, nil+}diff --git a/backend/plugin/db/doris/doris_test.go b/backend/plugin/db/doris/doris_test.gonew file mode 100644index 0000000000..178db87cdd--- /dev/null+++ b/backend/plugin/db/doris/doris_test.go@@ -0,0 +1,87 @@+package doris++import (+ "testing"++ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"++ // Register the Doris editor validator.+ _ "github.com/bytebase/bytebase/backend/plugin/parser/doris"+)++func TestParseVersion(t *testing.T) {+ tests := []struct {+ version string+ want string+ wantRest string+ }{+ {+ version: "8.0.27",+ want: "8.0.27",+ wantRest: "",+ },+ {+ version: "5.7.22-log",+ want: "5.7.22",+ wantRest: "-log",+ },+ {+ version: "5.6.29_ddm_3.0.1.7",+ want: "5.6.29",+ wantRest: "_ddm_3.0.1.7",+ },+ {+ version: "10.4.7-MariaDB",+ want: "10.4.7",+ wantRest: "-MariaDB",+ },+ }++ a := require.New(t)+ for _, tc := range tests {+ version, rest, err := parseVersion(tc.version)+ a.NoError(err)+ a.Equal(tc.want, version)+ a.Equal(tc.wantRest, rest)+ }+}++func TestContainsDelimiterDirective(t *testing.T) {+ tests := []struct {+ name string+ sql string+ want bool+ }{+ {"basic directive", "DELIMITER ;;\nCREATE PROCEDURE p() BEGIN SELECT 1; END;;\nDELIMITER ;", true},+ {"indented directive", " DELIMITER //\nSELECT 1//\nDELIMITER ;", true},+ {"tab-indented directive", "\tDELIMITER $$\n", true},+ {"string literal not a directive", "INSERT INTO t VALUES ('DELIMITER ');\nSELECT 1;", false},+ {"comment not a directive", "-- DELIMITER //\nSELECT 1;", false},+ {"column name not a directive", "SELECT delimiter FROM t;", false},+ {"no delimiter at all", "SELECT 1;\nSELECT 2;", false},+ }+ for _, tt := range tests {+ t.Run(tt.name, func(t *testing.T) {+ got := containsDelimiterDirective(tt.sql)+ require.Equal(t, tt.want, got)+ })+ }+}++// TestValidateSQLForEditorUsesDorisGrammar pins this driver to the Doris parser: a+// StarRocks-only spelling must be refused rather than quietly accepted, which is what+// routing the connection through a sibling engine's grammar would do.+func TestValidateSQLForEditorUsesDorisGrammar(t *testing.T) {+ // StarRocks generated column AS (expr); Doris wants GENERATED ALWAYS AS.+ const starRocksOnlyDDL = "CREATE TABLE t (a INT, b INT AS (a + 1)) DUPLICATE KEY(a) DISTRIBUTED BY HASH(a)"+ _, _, err := base.ValidateSQLForEditor(storepb.Engine_DORIS, starRocksOnlyDDL)+ require.Error(t, err)++ // A Doris CREATE TABLE parses, and is DDL rather than a read-only query.+ _, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_DORIS, "CREATE TABLE t (a INT) DUPLICATE KEY(a) DISTRIBUTED BY HASH(a)")+ require.NoError(t, err)+ require.False(t, allQuery)+}diff --git a/backend/plugin/db/doris/dump.go b/backend/plugin/db/doris/dump.gonew file mode 100644index 0000000000..7c70a54298--- /dev/null+++ b/backend/plugin/db/doris/dump.go@@ -0,0 +1,387 @@+package doris++import (+ "context"+ "database/sql"+ "fmt"+ "io"+ "log/slog"+ "strings"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/go-sql-driver/mysql"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common"+)++const (+ // MySQL error code for "Unknown table".+ errUnknownTable = 1051+)++// Dump and restore.+const (+ tableStmtFmt = "" ++ "--\n" ++ "-- Table structure for `%s`\n" ++ "--\n" ++ "%s;\n"+ viewStmtFmt = "" ++ "--\n" ++ "-- View structure for `%s`\n" ++ "--\n" ++ "%s;\n"+ materializedViewStmtFmt = "" ++ "--\n" ++ "-- Materialized view structure for `%s`\n" ++ "--\n" ++ "%s;\n"+ tempViewStmtFmt = "" ++ "--\n" ++ "-- Temporary view structure for `%s`\n" ++ "--\n" ++ "%s\n"+ tempMaterializedViewStmtFmt = "" ++ "--\n" ++ "-- Temporary view structure for materialized view `%s`\n" ++ "--\n" ++ "%s\n"+ disableUniqueAndForeignKeyCheckStmt = "SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;\nSET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;\n"+ restoreUniqueAndForeignKeyCheckStmt = "SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;\nSET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;\n"+)++// Dump dumps the database.+func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {+ slog.Debug("begin to dump database", slog.String("database", d.databaseName))++ db := d.GetDB()+ database := d.databaseName++ // Disable foreign key check.+ // mysqldump uses the same mechanism. When there is any schema or data dependency, we have to disable+ // the unique and foreign key check so that the restoring will not fail.+ if _, err := io.WriteString(out, disableUniqueAndForeignKeyCheckStmt); err != nil {+ return err+ }++ // Table and view statement.+ // We have to dump the table before views because of the structure dependency.+ tables, err := getTables(db, database)+ if err != nil {+ return errors.Wrapf(err, "failed to get tables of database %q", database)+ }+ // Construct temporal views and materialized views.+ // Create a temporary view with the same name as the view and with columns of+ // the same name in order to satisfy views that depend on this view.+ // This temporary view will be removed when the actual view is created.+ // The properties of each column, are not preserved in this temporary+ // view. They are not necessary because other views only need to reference+ // the column name, thus we generate SELECT 1 AS colName1, 1 AS colName2.+ // This will not be necessary once we can determine dependencies+ // between views and can simply dump them in the appropriate order.+ // https://sourcegraph.com/github.com/mysql/mysql-server/-/blob/client/mysqldump.cc?L2781+ for _, tbl := range tables {+ if tbl.TableType != viewTableType && tbl.TableType != materializedViewType {+ continue+ }+ if tbl.InvalidView != "" {+ // We will write the invalid view error string to schema.+ stmtFmt := viewStmtFmt+ if tbl.TableType == materializedViewType {+ stmtFmt = materializedViewStmtFmt+ }+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", fmt.Sprintf(stmtFmt, tbl.Name, fmt.Sprintf("-- %s", tbl.InvalidView)))); err != nil {+ return err+ }+ } else {+ tempView := getTemporaryView(tbl.Name, tbl.ViewColumns)+ if tbl.TableType == materializedViewType {+ tempView = getTemporaryMaterializedView(tbl.Name, tbl.ViewColumns)+ }+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tempView)); err != nil {+ return err+ }+ }+ }+ // Construct tables.+ for _, tbl := range tables {+ if tbl.TableType == viewTableType || tbl.TableType == materializedViewType {+ continue+ }+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.Statement)); err != nil {+ return err+ }+ }+ // Construct final views, then materialized views (see finalEmitOrder: an MV is+ // materialized from its sources at CREATE time, so its source views must be real+ // first, not the temporary SELECT 1 placeholders).+ for _, tbl := range finalEmitOrder(tables) {+ // The temporary placeholder created above is always a regular view (even for+ // materialized views — see getTemporaryMaterializedView), so it must be dropped+ // with DROP VIEW. DROP MATERIALIZED VIEW would not match the placeholder, leaving+ // it in the shared namespace so the real CREATE MATERIALIZED VIEW below collides.+ if _, err := io.WriteString(out, dropPlaceholderStmt(tbl.Name)); err != nil {+ return err+ }+ if _, err := io.WriteString(out, fmt.Sprintf("%s\n", tbl.Statement)); err != nil {+ return err+ }+ }++ // Restore foreign key check.+ if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {+ return err+ }++ return nil+}++// finalEmitOrder returns the views and materialized views in the order their real+// definitions must be emitted: all regular views first, then materialized views, each+// group keeping its original relative order. information_schema.tables is unordered, and+// a materialized view can be defined on a regular view and is materialized from its+// sources at CREATE time, so its source views must already be real (not the temporary+// SELECT 1 placeholders). Base tables are emitted separately and are excluded here.+//+// MVs keep their incoming order relative to each other, so a materialized view built on+// another materialized view (nested async MVs) is not yet dependency-ordered and could+// restore inactive if listed before its parent — tracked in BYT-9718.+func finalEmitOrder(tables []*TableSchema) []*TableSchema {+ var views, materializedViews []*TableSchema+ for _, tbl := range tables {+ switch tbl.TableType {+ case viewTableType:+ views = append(views, tbl)+ case materializedViewType:+ materializedViews = append(materializedViews, tbl)+ default:+ // Base tables and any other types are emitted elsewhere; skip here.+ }+ }+ return append(views, materializedViews...)+}++// dropPlaceholderStmt returns the statement that drops the temporary placeholder emitted+// for a view or materialized view before its real definition. The placeholder is always a+// regular view (see getTemporaryMaterializedView), so it is dropped with DROP VIEW+// regardless of the final object type.+func dropPlaceholderStmt(name string) string {+ return fmt.Sprintf("DROP VIEW IF EXISTS `%s`;\n", name)+}++func getTemporaryView(name string, columns []string) string {+ var parts []string+ for _, col := range columns {+ parts = append(parts, fmt.Sprintf("1 AS `%s`", col))+ }+ stmt := fmt.Sprintf("CREATE VIEW `%s` AS SELECT\n %s;\n", name, strings.Join(parts, ",\n "))+ return fmt.Sprintf(tempViewStmtFmt, name, stmt)+}++func getTemporaryMaterializedView(name string, columns []string) string {+ var parts []string+ for _, col := range columns {+ parts = append(parts, fmt.Sprintf("1 AS `%s`", col))+ }+ // For materialized views, we create a temporary regular view as a placeholder+ // since the actual materialized view will be created later with the full definition.+ stmt := fmt.Sprintf("CREATE VIEW `%s` AS SELECT\n %s;\n", name, strings.Join(parts, ",\n "))+ return fmt.Sprintf(tempMaterializedViewStmtFmt, name, stmt)+}++// TableSchema describes the schema of a table or view.+type TableSchema struct {+ Name string+ TableType string+ Statement string+ ViewColumns []string+ // InvalidView is the error message indicating an invalid view object.+ InvalidView string+}++// getTables gets all tables of a database using the provided database instance.+func getTables(db *sql.DB, dbName string) ([]*TableSchema, error) {+ var tables []*TableSchema+ query := "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?"+ rows, err := db.Query(query, dbName)+ if err != nil {+ return nil, err+ }+ defer rows.Close()++ columns, err := rows.Columns()+ if err != nil {+ return nil, err+ }+ for rows.Next() {+ var tbl TableSchema+ // Some versions add a third, unused storage-format column.+ var unusedStorageFormat string+ if len(columns) == 3 {+ if err := rows.Scan(&tbl.Name, &tbl.TableType, &unusedStorageFormat); err != nil {+ return nil, err+ }+ } else {+ if err := rows.Scan(&tbl.Name, &tbl.TableType); err != nil {+ return nil, err+ }+ }+ tables = append(tables, &tbl)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }++ // Materialized views need a separate catalog lookup and re-tagging so the dump emits+ // SHOW CREATE MATERIALIZED VIEW for them, since Doris reports them as BASE TABLE.+ materializedViewNames, err := getMaterializedViewNames(db, dbName)+ if err != nil {+ return nil, err+ }+ markMaterializedViews(tables, materializedViewNames)++ var result []*TableSchema+ for _, tbl := range tables {+ stmt, err := getTableStmt(db, dbName, tbl.Name, tbl.TableType)+ if err != nil {+ // Skip tables that were dropped between listing and fetching CREATE statement.+ var mysqlErr *mysql.MySQLError+ if errors.As(err, &mysqlErr) && mysqlErr.Number == errUnknownTable {+ slog.Debug("table was dropped during dump, skipping",+ slog.String("database", dbName),+ slog.String("table", tbl.Name))+ continue+ }+ return nil, errors.Wrapf(err, "failed to call getTableStmt(%q, %q, %q)", dbName, tbl.Name, tbl.TableType)+ }+ tbl.Statement = stmt+ if tbl.TableType == viewTableType || tbl.TableType == materializedViewType {+ viewColumns, err := getViewColumns(db, dbName, tbl.Name)+ if err != nil {+ tbl.InvalidView = err.Error()+ } else {+ tbl.ViewColumns = viewColumns+ }+ }+ result = append(result, tbl)+ }+ return result, nil+}++// getMaterializedViewNames returns the set of materialized-view names in the database.+// The query is best-effort: an unsupported version logs and yields an empty set.+func getMaterializedViewNames(db *sql.DB, dbName string) (map[string]bool, error) {+ names := make(map[string]bool)++ query := fmt.Sprintf(`SELECT Name FROM mv_infos("database"="%s")`, dbName)++ rows, err := db.Query(query)+ if err != nil {+ // The catalog may be unavailable on older engine versions; log and continue.+ slog.Debug("failed to query materialized views, might not be supported", slog.String("error", err.Error()))+ return names, nil+ }+ defer rows.Close()++ for rows.Next() {+ var name string+ if err := rows.Scan(&name); err != nil {+ return nil, err+ }+ names[name] = true+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return names, nil+}++// markMaterializedViews re-tags rows that are actually materialized views so the dump+// emits SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE', which+// becomes 'MATERIALIZED VIEW' when the name is in the materialized-view set. Regular+// tables and views are left untouched.+func markMaterializedViews(tables []*TableSchema, materializedViewNames map[string]bool) {+ for _, tbl := range tables {+ if !materializedViewNames[tbl.Name] {+ continue+ }+ if tbl.TableType == baseTableType || tbl.TableType == viewTableType {+ tbl.TableType = materializedViewType+ }+ }+}++// getTableStmt gets the create statement of a table.+func getTableStmt(db *sql.DB, dbName, tblName, tblType string) (string, error) {+ switch tblType {+ case baseTableType:+ query := fmt.Sprintf("SHOW CREATE TABLE `%s`.`%s`;", dbName, tblName)+ var stmt, unused string+ if err := db.QueryRow(query).Scan(&unused, &stmt); err != nil {+ if err == sql.ErrNoRows {+ return "", common.FormatDBErrorEmptyRowWithQuery(query)+ }+ return "", err+ }+ // Remove trailing semicolon if present, as the format string will add one+ stmt = strings.TrimSuffix(stmt, ";")+ return fmt.Sprintf(tableStmtFmt, tblName, stmt), nil+ case viewTableType:+ // This differs from mysqldump as it includes.+ query := fmt.Sprintf("SHOW CREATE VIEW `%s`.`%s`;", dbName, tblName)+ var createStmt, unused string+ if err := db.QueryRow(query).Scan(&unused, &createStmt, &unused, &unused); err != nil {+ if err == sql.ErrNoRows {+ return "", common.FormatDBErrorEmptyRowWithQuery(query)+ }+ return "", err+ }+ // Remove trailing semicolon if present, as the format string will add one+ createStmt = strings.TrimSuffix(createStmt, ";")+ return fmt.Sprintf(viewStmtFmt, tblName, createStmt), nil+ case materializedViewType:+ // For materialized views we use SHOW CREATE MATERIALIZED VIEW.+ // This command returns 2 columns: materialized view name and create statement.+ query := fmt.Sprintf("SHOW CREATE MATERIALIZED VIEW `%s`.`%s`;", dbName, tblName)+ var createStmt, unused string+ if err := db.QueryRow(query).Scan(&unused, &createStmt); err != nil {+ if err == sql.ErrNoRows {+ return "", common.FormatDBErrorEmptyRowWithQuery(query)+ }+ return "", err+ }+ // Remove trailing semicolon if present, as the format string will add one+ createStmt = strings.TrimSuffix(createStmt, ";")+ return fmt.Sprintf(materializedViewStmtFmt, tblName, createStmt), nil+ default:+ slog.Debug("skip unsupported table type", slog.String("dbName", dbName), slog.String("tableName", tblName), slog.String("tableType", tblType))+ return "", nil+ }+}++// getViewColumns gets the create statement of a table.+func getViewColumns(db *sql.DB, dbName, tblName string) ([]string, error) {+ query := fmt.Sprintf("SHOW COLUMNS FROM `%s`.`%s`;", dbName, tblName)+ // https://dev.mysql.com/doc/refman/8.0/en/show-columns.html+ // Field, Type, Null, Key, Default, Extra.+ rows, err := db.Query(query)+ if err != nil {+ return nil, err+ }+ defer rows.Close()++ var fields []string+ for rows.Next() {+ var field string+ var unused sql.NullString+ if err := rows.Scan(&field, &unused, &unused, &unused, &unused, &unused); err != nil {+ return nil, err+ }+ fields = append(fields, field)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }+ return fields, nil+}diff --git a/backend/plugin/db/doris/dump_routine_probe_test.go b/backend/plugin/db/doris/dump_routine_probe_test.gonew file mode 100644index 0000000000..6443433ed3--- /dev/null+++ b/backend/plugin/db/doris/dump_routine_probe_test.go@@ -0,0 +1,143 @@+package doris++import (+ "bytes"+ "context"+ "database/sql"+ "database/sql/driver"+ "fmt"+ "io"+ "regexp"+ "strings"+ "sync"+ "testing"++ "github.com/go-sql-driver/mysql"+ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// BYT-10211: Doris has no MySQL stored routines or events, and Doris+// kernels that dropped the PL/SQL grammar (SelectDB 4.0.12+/4.1.8+/5.0.5+, apache/doris#58700)+// reject `SHOW FUNCTION STATUS` outright, which aborted the whole schema sync. The dump must+// complete without ever sending the MySQL routine or event probes.+func TestDumpSendsNoRoutineOrEventProbe(t *testing.T) {+ dumpRecorder.reset(plsqlStrippedCatalog())+ sqlDB, err := sql.Open(dumpRecorderDriverName, "")+ require.NoError(t, err)+ defer sqlDB.Close()++ d := &Driver{dbType: storepb.Engine_DORIS, db: sqlDB, databaseName: "analytics"}+ var out bytes.Buffer+ require.NoError(t, d.Dump(context.Background(), &out, nil))++ require.Contains(t, out.String(), "CREATE TABLE `orders`")+ require.Contains(t, out.String(), "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`")+ for _, q := range dumpRecorder.recorded() {+ require.NotRegexp(t, forbiddenProbe, q)+ }+}++var (+ routineStatusProbe = regexp.MustCompile(`(?i)^\s*SHOW\s+(FUNCTION|PROCEDURE)\s+STATUS`)+ forbiddenProbe = regexp.MustCompile(`(?i)SHOW\s+((FUNCTION|PROCEDURE)\s+STATUS|EVENTS|CREATE\s+(FUNCTION|PROCEDURE|EVENT))`)+)++// plsqlStrippedCatalog answers the dump's table and view queries with the column shapes a+// live Doris 4.1.3 returns, and rejects the routine probes with the exact error a+// PL/SQL-stripped kernel returns.+func plsqlStrippedCatalog() func(string) ([]string, [][]driver.Value, error) {+ return func(query string) ([]string, [][]driver.Value, error) {+ switch {+ case routineStatusProbe.MatchString(query):+ return nil, nil, &mysql.MySQLError{Number: 1105, SQLState: [5]byte{'H', 'Y', '0', '0', '0'}, Message: "errCode = 2, detailMessage = no viable alternative at input 'SHOW FUNCTION'(line 1, pos 5)"}+ case strings.Contains(query, "information_schema.TABLES"):+ return []string{"TABLE_NAME", "TABLE_TYPE"}, [][]driver.Value{{"orders", "BASE TABLE"}, {"v_orders", "VIEW"}}, nil+ case strings.Contains(query, "mv_infos("):+ return []string{"Name"}, nil, nil+ case strings.HasPrefix(query, "SHOW CREATE TABLE"):+ return []string{"Table", "Create Table"}, [][]driver.Value{{"orders", "CREATE TABLE `orders` (\n `id` int NULL\n) ENGINE=OLAP"}}, nil+ case strings.HasPrefix(query, "SHOW CREATE VIEW"):+ return []string{"View", "Create View", "character_set_client", "collation_connection"}, [][]driver.Value{{"v_orders", "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`", "utf8", "utf8_general_ci"}}, nil+ case strings.HasPrefix(query, "SHOW COLUMNS FROM"):+ return []string{"Field", "Type", "Null", "Key", "Default", "Extra"}, [][]driver.Value{{"id", "int", "YES", "", nil, ""}}, nil+ }+ return nil, nil, errors.Errorf("unexpected statement: %q", query)+ }+}++// dumpRecorder is registered once: database/sql panics on a repeated driver name, which+// would take the whole package down under `go test -count=2`. Each subtest resets it.+const dumpRecorderDriverName = "doris-dump-recorder"++var dumpRecorder recordingDriver++func init() {+ sql.Register(dumpRecorderDriverName, &dumpRecorder)+}++// recordingDriver is a database/sql driver that records every statement and answers it+// from a canned function, so Dump runs against a scripted server without a network.+type recordingDriver struct {+ mu sync.Mutex+ queries []string+ answer func(query string) ([]string, [][]driver.Value, error)+}++func (d *recordingDriver) Open(string) (driver.Conn, error) { return &recordingConn{d: d}, nil }++func (d *recordingDriver) reset(answer func(string) ([]string, [][]driver.Value, error)) {+ d.mu.Lock()+ defer d.mu.Unlock()+ d.queries = nil+ d.answer = answer+}++func (d *recordingDriver) recorded() []string {+ d.mu.Lock()+ defer d.mu.Unlock()+ return append([]string(nil), d.queries...)+}++type recordingConn struct{ d *recordingDriver }++func (*recordingConn) Prepare(string) (driver.Stmt, error) {+ return nil, errors.New("prepare is not supported")+}+func (*recordingConn) Close() error { return nil }+func (*recordingConn) Begin() (driver.Tx, error) { return nil, errors.New("tx is not supported") }++// QueryContext makes database/sql skip Prepare for both db.Query(q) and db.Query(q, arg);+// the driver interpolates the argument itself, as the real DSN (interpolateParams=true) does.+func (c *recordingConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {+ for _, a := range args {+ query = strings.Replace(query, "?", fmt.Sprintf("'%v'", a.Value), 1)+ }+ c.d.mu.Lock()+ c.d.queries = append(c.d.queries, query)+ c.d.mu.Unlock()+ cols, rows, err := c.d.answer(query)+ if err != nil {+ return nil, err+ }+ return &sliceRows{cols: cols, rows: rows}, nil+}++type sliceRows struct {+ cols []string+ rows [][]driver.Value+ i int+}++func (r *sliceRows) Columns() []string { return r.cols }+func (*sliceRows) Close() error { return nil }+func (r *sliceRows) Next(dest []driver.Value) error {+ if r.i >= len(r.rows) {+ return io.EOF+ }+ copy(dest, r.rows[r.i])+ r.i+++ return nil+}diff --git a/backend/plugin/db/doris/dump_test.go b/backend/plugin/db/doris/dump_test.gonew file mode 100644index 0000000000..0739b855df--- /dev/null+++ b/backend/plugin/db/doris/dump_test.go@@ -0,0 +1,67 @@+package doris++import (+ "testing"++ "github.com/stretchr/testify/require"+)++func TestGetTemporaryView(t *testing.T) {+ a := require.New(t)+ got := getTemporaryView("db1", []string{"col1", "col2"})+ want := "--\n-- Temporary view structure for `db1`\n--\nCREATE VIEW `db1` AS SELECT\n 1 AS `col1`,\n 1 AS `col2`;\n\n"+ a.Equal(want, got)+}++// BYT-9689: the dump path re-tags materialized-view rows so it emits+// SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE', and some+// setups as 'VIEW'; either must become 'MATERIALIZED VIEW' when the name is a known MV,+// while regular tables and views are left untouched.+func TestMarkMaterializedViews(t *testing.T) {+ tables := []*TableSchema{+ {Name: "mv_base", TableType: baseTableType}, // MV reported as BASE TABLE+ {Name: "mv_view", TableType: viewTableType}, // MV reported as VIEW+ {Name: "v_regular", TableType: viewTableType},+ {Name: "t_plain", TableType: baseTableType},+ }+ markMaterializedViews(tables, map[string]bool{"mv_base": true, "mv_view": true})++ require.Equal(t, materializedViewType, tables[0].TableType)+ require.Equal(t, materializedViewType, tables[1].TableType)+ require.Equal(t, viewTableType, tables[2].TableType, "regular view must stay a view")+ require.Equal(t, baseTableType, tables[3].TableType, "plain table must stay a table")+}++// BYT-9689: the dump emits a temporary regular-view placeholder for every view AND+// materialized view (getTemporaryMaterializedView creates a CREATE VIEW), then drops it+// before emitting the real definition. The drop must therefore be DROP VIEW even for an+// MV — DROP MATERIALIZED VIEW would not match the placeholder, leaving it in the shared+// namespace so the real CREATE MATERIALIZED VIEW collides with it on replay.+func TestMaterializedViewPlaceholderIsDroppedAsView(t *testing.T) {+ placeholder := getTemporaryMaterializedView("mv1", []string{"a"})+ require.Contains(t, placeholder, "CREATE VIEW `mv1`")+ require.NotContains(t, placeholder, "CREATE MATERIALIZED VIEW")++ require.Equal(t, "DROP VIEW IF EXISTS `mv1`;\n", dropPlaceholderStmt("mv1"))+}++// BYT-9689: information_schema.tables is unordered, so an MV can be listed before a view+// it is defined on. A materialized view is materialized from its sources at CREATE time+// (unlike a lazy regular view), so all regular views must be emitted — and on restore,+// created real — before any materialized view; otherwise the MV would materialize from+// the temporary SELECT 1 placeholder. finalEmitOrder partitions views ahead of MVs while+// preserving each group's relative order, and drops base tables (emitted elsewhere).+func TestFinalEmitOrder_ViewsBeforeMaterializedViews(t *testing.T) {+ tables := []*TableSchema{+ {Name: "mv1", TableType: materializedViewType},+ {Name: "v1", TableType: viewTableType},+ {Name: "t1", TableType: baseTableType},+ {Name: "mv2", TableType: materializedViewType},+ {Name: "v2", TableType: viewTableType},+ }+ var names []string+ for _, tbl := range finalEmitOrder(tables) {+ names = append(names, tbl.Name)+ }+ require.Equal(t, []string{"v1", "v2", "mv1", "mv2"}, names)+}diff --git a/backend/plugin/db/doris/query.go b/backend/plugin/db/doris/query.gonew file mode 100644index 0000000000..4e5150015a--- /dev/null+++ b/backend/plugin/db/doris/query.go@@ -0,0 +1,76 @@+package doris++import (+ "database/sql"++ v1pb "github.com/bytebase/bytebase/backend/generated-go/v1"+ "github.com/bytebase/bytebase/backend/plugin/db/util"+)++func makeValueByTypeName(typeName string, _ *sql.ColumnType) any {+ switch typeName {+ case "VARCHAR", "TEXT", "UUID", "DATETIME", "TIMESTAMP":+ return new(sql.NullString)+ case "BOOL":+ return new(sql.NullBool)+ case "INT", "INTEGER", "TINYINT", "SMALLINT", "MEDIUMINT", "BIGINT", "INT2", "INT4", "INT8":+ return new(sql.NullInt64)+ case "FLOAT", "DOUBLE", "FLOAT4", "FLOAT8":+ return new(sql.NullFloat64)+ case "BIT", "VARBIT":+ return new([]byte)+ default:+ return new(sql.NullString)+ }+}++func convertValue(typeName string, columnType *sql.ColumnType, value any) *v1pb.RowValue {+ switch raw := value.(type) {+ case *sql.NullString:+ if raw.Valid {+ if typeName == "TIMESTAMP" || typeName == "DATETIME" {+ _, scale, _ := columnType.DecimalSize()+ return util.BuildTimestampOrStringRowValue(raw.String, scale)+ }+ return &v1pb.RowValue{+ Kind: &v1pb.RowValue_StringValue{+ StringValue: raw.String,+ },+ }+ }+ case *sql.NullInt64:+ if raw.Valid {+ return &v1pb.RowValue{+ Kind: &v1pb.RowValue_Int64Value{+ Int64Value: raw.Int64,+ },+ }+ }+ case *[]byte:+ if len(*raw) > 0 {+ return &v1pb.RowValue{+ Kind: &v1pb.RowValue_BytesValue{+ BytesValue: *raw,+ },+ }+ }+ case *sql.NullBool:+ if raw.Valid {+ return &v1pb.RowValue{+ Kind: &v1pb.RowValue_BoolValue{+ BoolValue: raw.Bool,+ },+ }+ }+ case *sql.NullFloat64:+ if raw.Valid {+ return &v1pb.RowValue{+ Kind: &v1pb.RowValue_DoubleValue{+ DoubleValue: raw.Float64,+ },+ }+ }+ default:+ }+ return util.NullRowValue+}diff --git a/backend/plugin/db/doris/sync.go b/backend/plugin/db/doris/sync.gonew file mode 100644index 0000000000..001e309248--- /dev/null+++ b/backend/plugin/db/doris/sync.go@@ -0,0 +1,387 @@+package doris++import (+ "context"+ "database/sql"+ "fmt"+ "log/slog"+ "strconv"+ "strings"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/pkg/errors"++ "github.com/bytebase/bytebase/backend/common"+ "github.com/bytebase/bytebase/backend/common/log"+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/db"+ "github.com/bytebase/bytebase/backend/plugin/db/mysql"+ "github.com/bytebase/bytebase/backend/plugin/db/util"+)++const (+ autoIncrementSymbol = "AUTO_INCREMENT"+)++// systemDatabaseExclusion returns the quoted, comma-joined system databases to+// exclude from instance discovery, matching the parser-side system-database+// classification. Doris has no `sys` metadatabase.+func systemDatabaseExclusion() string {+ return strings.Join([]string{"'information_schema'", "'_statistics_'"}, ", ")+}++// SyncInstance syncs the instance.+func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error) {+ version, _, err := d.getVersion(ctx)+ if err != nil {+ return nil, err+ }++ var lowerCaseTableNames int32+ lowerCaseTableNamesText, err := d.getServerVariable(ctx, "lower_case_table_names")+ if err != nil {+ slog.Debug("failed to get lower_case_table_names variable", log.BBError(err))+ } else {+ v, err := strconv.ParseInt(lowerCaseTableNamesText, 10, 32)+ if err != nil {+ slog.Debug("failed to parse lower_case_table_names variable", log.BBError(err))+ } else {+ lowerCaseTableNames = int32(v)+ }+ }++ // Query db info+ where := fmt.Sprintf("LOWER(SCHEMA_NAME) NOT IN (%s)", systemDatabaseExclusion())+ query := `+ SELECT+ SCHEMA_NAME,+ DEFAULT_CHARACTER_SET_NAME,+ DEFAULT_COLLATION_NAME+ FROM information_schema.SCHEMATA+ WHERE ` + where+ rows, err := d.db.QueryContext(ctx, query)+ if err != nil {+ return nil, util.FormatErrorWithQuery(err, query)+ }+ defer rows.Close()++ var databases []*metadatapb.DatabaseSchemaMetadata+ for rows.Next() {+ database := &metadatapb.DatabaseSchemaMetadata{}+ if err := rows.Scan(+ &database.Name,+ &database.CharacterSet,+ &database.Collation,+ ); err != nil {+ return nil, err+ }+ database.CharacterSet = ""+ database.Collation = ""+ databases = append(databases, database)+ }+ if err := rows.Err(); err != nil {+ return nil, err+ }++ return &db.InstanceMetadata{+ Version: version,+ Databases: databases,+ Metadata: &storepb.Instance{+ MysqlLowerCaseTableNames: lowerCaseTableNames,+ },+ }, nil+}++func (d *Driver) getServerVariable(ctx context.Context, varName string) (string, error) {+ db := d.GetDB()+ query := fmt.Sprintf("SHOW VARIABLES LIKE '%s'", varName)+ var varNameFound, value string+ if err := db.QueryRowContext(ctx, query).Scan(&varNameFound, &value); err != nil {+ if err == sql.ErrNoRows {+ return "", common.FormatDBErrorEmptyRowWithQuery(query)+ }+ return "", util.FormatErrorWithQuery(err, query)+ }+ if varName != varNameFound {+ return "", errors.Errorf("expecting variable %s, but got %s", varName, varNameFound)+ }+ return value, nil+}++// isMaterializedView reports whether a table row is a materialized view, based on its+// table type and membership in the materialized-view set. Doris reports materialized+// views with TABLE_TYPE='BASE TABLE', and some setups with 'MATERIALIZED VIEW'; in+// either case, presence in materializedViewMap is the authoritative signal.+func isMaterializedView(tableType string, key db.TableKey, materializedViewMap map[db.TableKey]*metadatapb.MaterializedViewMetadata) bool {+ if _, ok := materializedViewMap[key]; !ok {+ return false+ }+ switch tableType {+ case baseTableType, viewTableType, materializedViewType:+ return true+ default:+ return false+ }+}++// getMaterializedViews returns the materialized views of the current database keyed by+// table name. The query is best-effort: a version that does not support it is logged+// and yields an empty set rather than failing the sync.+func (d *Driver) getMaterializedViews(ctx context.Context) (map[db.TableKey]*metadatapb.MaterializedViewMetadata, error) {+ materializedViewMap := make(map[db.TableKey]*metadatapb.MaterializedViewMetadata)++ // Doris exposes materialized views via the mv_infos() table-valued function.+ // https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/mv_infos+ query := fmt.Sprintf(`SELECT Name, QuerySql FROM mv_infos("database"="%s")`, d.databaseName)++ rows, err := d.db.QueryContext(ctx, query)+ if err != nil {+ // The catalog may be unavailable on older engine versions; log and continue.+ slog.Debug("failed to query materialized views, might not be supported in this version", log.BBError(err))+ return materializedViewMap, nil+ }+ defer rows.Close()++ for rows.Next() {+ materializedView := &metadatapb.MaterializedViewMetadata{}+ if err := rows.Scan(&materializedView.Name, &materializedView.Definition); err != nil {+ return nil, err+ }+ key := db.TableKey{Schema: "", Table: materializedView.Name}+ materializedViewMap[key] = materializedView+ }+ if err := rows.Err(); err != nil {+ return nil, util.FormatErrorWithQuery(err, query)+ }+ return materializedViewMap, nil+}++// SyncDBSchema syncs a single database schema.+func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMetadata, error) {+ schemaMetadata := &metadatapb.SchemaMetadata{+ Name: "",+ }++ // There is not yet a way to list indexes from information_schema.+ // Query column info.+ columnMap := make(map[db.TableKey][]*metadatapb.ColumnMetadata)+ columnQuery := fmt.Sprintf(`+ SELECT+ TABLE_NAME,+ IFNULL(COLUMN_NAME, ''),+ ORDINAL_POSITION,+ COLUMN_DEFAULT,+ IS_NULLABLE,+ COLUMN_TYPE,+ IFNULL(CHARACTER_SET_NAME, ''),+ IFNULL(COLLATION_NAME, ''),+ COLUMN_COMMENT,+ EXTRA+ FROM information_schema.columns+ WHERE TABLE_SCHEMA = '%s'+ ORDER BY TABLE_NAME, ORDINAL_POSITION`, d.databaseName)+ columnRows, err := d.db.QueryContext(ctx, columnQuery)+ if err != nil {+ return nil, util.FormatErrorWithQuery(err, columnQuery)+ }+ defer columnRows.Close()+ for columnRows.Next() {+ column := &metadatapb.ColumnMetadata{}+ var tableName, nullable, extra string+ var defaultStr sql.NullString+ if err := columnRows.Scan(+ &tableName,+ &column.Name,+ &column.Position,+ &defaultStr,+ &nullable,+ &column.Type,+ &column.CharacterSet,+ &column.Collation,+ &column.Comment,+ &extra,+ ); err != nil {+ return nil, err+ }+ isNullBool, err := util.ConvertYesNo(nullable)+ if err != nil {+ return nil, err+ }+ if defaultStr.Valid {+ // Doris is MySQL-compatible, so use MySQL's default handling approach+ unquotedDefault := mysql.UnquoteMySQLString(defaultStr.String)+ switch {+ case mysql.IsCurrentTimestampLike(unquotedDefault):+ column.Default = unquotedDefault+ case strings.Contains(extra, "DEFAULT_GENERATED"):+ unescapedDefault := mysql.UnescapeExpressionDefault(unquotedDefault)+ column.Default = fmt.Sprintf("(%s)", unescapedDefault)+ default:+ // For non-generated and non CURRENT_XXX default value, preserve quotes for mysqldump compatibility+ column.Default = defaultStr.String+ }+ } else if strings.Contains(strings.ToUpper(extra), autoIncrementSymbol) {+ // TODO(zp): refactor column default value.+ // Use the upper case to consistent with MySQL Dump.+ column.Default = autoIncrementSymbol+ } else if isNullBool {+ // This is NULL if the column has an explicit default of NULL,+ // or if the column definition includes no DEFAULT clause.+ // https://dev.mysql.com/doc/refman/8.0/en/information-schema-columns-table.html+ column.Default = "NULL"+ }+ column.Nullable = isNullBool++ key := db.TableKey{Schema: "", Table: tableName}+ columnMap[key] = append(columnMap[key], column)+ }+ if err := columnRows.Err(); err != nil {+ return nil, util.FormatErrorWithQuery(err, columnQuery)+ }++ // Query view info.+ viewMap := make(map[db.TableKey]*metadatapb.ViewMetadata)+ viewQuery := fmt.Sprintf(`+ SELECT+ TABLE_NAME,+ VIEW_DEFINITION+ FROM information_schema.VIEWS+ WHERE TABLE_SCHEMA = '%s'`, d.databaseName)+ viewRows, err := d.db.QueryContext(ctx, viewQuery)+ if err != nil {+ return nil, util.FormatErrorWithQuery(err, viewQuery)+ }+ defer viewRows.Close()+ for viewRows.Next() {+ view := &metadatapb.ViewMetadata{}+ if err := viewRows.Scan(+ &view.Name,+ &view.Definition,+ ); err != nil {+ return nil, err+ }+ key := db.TableKey{Schema: "", Table: view.Name}+ view.Columns = columnMap[key]+ viewMap[key] = view+ }+ if err := viewRows.Err(); err != nil {+ return nil, util.FormatErrorWithQuery(err, viewQuery)+ }++ // Query materialized view info (engine-specific catalog; best-effort).+ materializedViewMap, err := d.getMaterializedViews(ctx)+ if err != nil {+ return nil, err+ }++ // Query table info.+ tableQuery := fmt.Sprintf(`+ SELECT+ TABLE_NAME,+ TABLE_TYPE,+ IFNULL(ENGINE, ''),+ IFNULL(TABLE_COLLATION, ''),+ IFNULL(TABLE_ROWS, 0),+ IFNULL(DATA_LENGTH, 0),+ IFNULL(INDEX_LENGTH, 0),+ IFNULL(DATA_FREE, 0),+ IFNULL(CREATE_OPTIONS, ''),+ IFNULL(TABLE_COMMENT, '')+ FROM information_schema.TABLES+ WHERE TABLE_SCHEMA = '%s'+ ORDER BY TABLE_NAME`, d.databaseName)+ tableRows, err := d.db.QueryContext(ctx, tableQuery)+ if err != nil {+ return nil, util.FormatErrorWithQuery(err, tableQuery)+ }+ defer tableRows.Close()+ for tableRows.Next() {+ var tableName, tableType, engine, collation, createOptions, comment string+ var rowCount, dataSize, indexSize, dataFree int64+ // Workaround TiDB bug https://github.com/pingcap/tidb/issues/27970+ var tableCollation sql.NullString+ if err := tableRows.Scan(+ &tableName,+ &tableType,+ &engine,+ &collation,+ &rowCount,+ &dataSize,+ &indexSize,+ &dataFree,+ &createOptions,+ &comment,+ ); err != nil {+ return nil, err+ }++ key := db.TableKey{Schema: "", Table: tableName}+ // Doris reports materialized views as TABLE_TYPE='BASE TABLE';+ // route any row whose name is in the materialized-view set to MaterializedViews.+ if isMaterializedView(tableType, key, materializedViewMap) {+ materializedView := materializedViewMap[key]+ materializedView.Comment = comment+ schemaMetadata.MaterializedViews = append(schemaMetadata.MaterializedViews, materializedView)+ continue+ }+ switch tableType {+ case baseTableType:+ tableMetadata := &metadatapb.TableMetadata{+ Name: tableName,+ Columns: columnMap[key],+ Engine: engine,+ Collation: collation,+ RowCount: rowCount,+ DataSize: dataSize,+ IndexSize: indexSize,+ DataFree: dataFree,+ CreateOptions: createOptions,+ Comment: comment,+ }+ if tableCollation.Valid {+ tableMetadata.Collation = tableCollation.String+ }+ // TODO(d): add index information whenever it is available.+ schemaMetadata.Tables = append(schemaMetadata.Tables, tableMetadata)+ case viewTableType:+ if view, ok := viewMap[key]; ok {+ view.Comment = comment+ schemaMetadata.Views = append(schemaMetadata.Views, view)+ }+ default:+ // Includes 'MATERIALIZED VIEW' rows absent from the materialized-view set+ // and any other unrecognized type.+ slog.Debug("skipping unhandled table type", slog.String("tableName", tableName), slog.String("tableType", tableType))+ }+ }+ if err := tableRows.Err(); err != nil {+ return nil, util.FormatErrorWithQuery(err, tableQuery)+ }++ databaseMetadata := &metadatapb.DatabaseSchemaMetadata{+ Name: d.databaseName,+ Schemas: []*metadatapb.SchemaMetadata{schemaMetadata},+ }+ // Query db info.+ databaseQuery := fmt.Sprintf(`+ SELECT+ DEFAULT_CHARACTER_SET_NAME,+ DEFAULT_COLLATION_NAME+ FROM information_schema.SCHEMATA+ WHERE SCHEMA_NAME = '%s'`, d.databaseName)+ if err := d.db.QueryRowContext(ctx, databaseQuery).Scan(+ &databaseMetadata.CharacterSet,+ &databaseMetadata.Collation,+ ); err != nil {+ if err == sql.ErrNoRows {+ return nil, common.Errorf(common.NotFound, "database %q not found", d.databaseName)+ }+ return nil, err+ }+ // "characterSet":"utf8\u0000", "collation":"utf8_general_ci\u0000".+ // ERROR: unsupported Unicode escape sequence (SQLSTATE 22P05).+ databaseMetadata.CharacterSet = ""+ databaseMetadata.Collation = ""++ return databaseMetadata, err+}diff --git a/backend/plugin/db/doris/sync_test.go b/backend/plugin/db/doris/sync_test.gonew file mode 100644index 0000000000..439fef85de--- /dev/null+++ b/backend/plugin/db/doris/sync_test.go@@ -0,0 +1,39 @@+package doris++import (+ "testing"++ metadatapb "github.com/bytebase/omni/metadata"+ "github.com/stretchr/testify/require"++ "github.com/bytebase/bytebase/backend/plugin/db"+)++// BYT-9689: Doris reports materialized views in information_schema.tables with+// TABLE_TYPE='BASE TABLE', and some setups with 'MATERIALIZED VIEW', so the table type+// alone cannot identify one. Membership in the materialized-view set is the+// authoritative signal, whatever type the row carries.+func TestIsMaterializedView(t *testing.T) {+ mvMap := map[db.TableKey]*metadatapb.MaterializedViewMetadata{+ {Schema: "", Table: "mv_async"}: {Name: "mv_async"},+ }+ mvKey := db.TableKey{Schema: "", Table: "mv_async"}++ require.True(t, isMaterializedView(baseTableType, mvKey, mvMap))+ require.True(t, isMaterializedView(materializedViewType, mvKey, mvMap))+ require.True(t, isMaterializedView(viewTableType, mvKey, mvMap))++ // A name absent from the MV set is never a materialized view.+ require.False(t, isMaterializedView(viewTableType, db.TableKey{Table: "v_regular"}, mvMap))+ require.False(t, isMaterializedView(baseTableType, db.TableKey{Table: "sales"}, mvMap))+}++// #20566 review (P2): SyncInstance's system-database exclusion must match the parser's+// system set. Doris has no `sys` metadatabase, so excluding it would risk hiding a user+// database named sys.+func TestSystemDatabaseExclusion(t *testing.T) {+ got := systemDatabaseExclusion()+ require.Contains(t, got, "'information_schema'")+ require.Contains(t, got, "'_statistics_'")+ require.NotContains(t, got, "'sys'")+}diff --git a/backend/plugin/db/starrocks/dump.go b/backend/plugin/db/starrocks/dump.goindex 412404eb78..de8fa71570 100644--- a/backend/plugin/db/starrocks/dump.go+++ b/backend/plugin/db/starrocks/dump.go@@ -13,7 +13,6 @@ import ("github.com/pkg/errors""github.com/bytebase/bytebase/backend/common"- storepb "github.com/bytebase/bytebase/backend/generated-go/store")const (@@ -48,8 +47,6 @@ const ("-- Temporary view structure for materialized view `%s`\n" +"--\n" +"%s\n"- disableUniqueAndForeignKeyCheckStmt = "SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;\nSET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;\n"- restoreUniqueAndForeignKeyCheckStmt = "SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;\nSET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;\n")// Dump dumps the database.@@ -58,23 +55,14 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseScdb := d.GetDB()database := d.databaseName- dbType := d.dbType- // Disable foreign key check.- // mysqldump uses the same mechanism. When there is any schema or data dependency, we have to disable- // the unique and foreign key check so that the restoring will not fail.- // StarRocks has no MySQL UNIQUE_CHECKS/FOREIGN_KEY_CHECKS session variables (and does not- // enforce foreign keys), so emitting the guard makes the dump fail to replay with- // "Unknown system variable 'UNIQUE_CHECKS'".- if d.dbType != storepb.Engine_STARROCKS {- if _, err := io.WriteString(out, disableUniqueAndForeignKeyCheckStmt); err != nil {- return err- }- }+ // mysqldump brackets a dump with UNIQUE_CHECKS/FOREIGN_KEY_CHECKS guards. StarRocks has+ // neither session variable and does not enforce foreign keys, so emitting them makes the+ // dump fail to replay with "Unknown system variable 'UNIQUE_CHECKS'".// Table and view statement.// We have to dump the table before views because of the structure dependency.- tables, err := getTables(db, database, dbType)+ tables, err := getTables(db, database)if err != nil {return errors.Wrapf(err, "failed to get tables of database %q", database)}@@ -136,13 +124,6 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSc}}- // Restore foreign key check (skipped for StarRocks, which lacks these variables).- if d.dbType != storepb.Engine_STARROCKS {- if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {- return err- }- }-return nil}@@ -210,7 +191,7 @@ type TableSchema struct {}// getTables gets all tables of a database using the provided database instance.-func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema, error) {+func getTables(db *sql.DB, dbName string) ([]*TableSchema, error) {var tables []*TableSchemaquery := "SELECT TABLE_NAME, TABLE_TYPE FROM information_schema.TABLES WHERE TABLE_SCHEMA = ?"rows, err := db.Query(query, dbName)@@ -225,10 +206,9 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema}for rows.Next() {var tbl TableSchema- // StorageFormat is the third but unused column for Doris.+ // Some versions add a third, unused storage-format column.var unusedStorageFormat stringif len(columns) == 3 {- // Doris.if err := rows.Scan(&tbl.Name, &tbl.TableType, &unusedStorageFormat); err != nil {return nil, err}@@ -244,8 +224,8 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema}// Materialized views need a separate catalog lookup and re-tagging so the dump emits- // SHOW CREATE MATERIALIZED VIEW for them (Doris reports them as BASE TABLE, StarRocks as VIEW).- materializedViewNames, err := getMaterializedViewNames(db, dbName, dbType)+ // SHOW CREATE MATERIALIZED VIEW for them, since StarRocks reports them as VIEW.+ materializedViewNames, err := getMaterializedViewNames(db, dbName)if err != nil {return nil, err}@@ -280,23 +260,14 @@ func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema}// getMaterializedViewNames returns the set of materialized-view names in the database.-// Doris and StarRocks expose them through different catalogs; the query is best-effort-// (an unsupported engine/version logs and yields an empty set). StarRocks sync rollups-// (REFRESH_TYPE='ROLLUP') are excluded — they are rollup indexes on the base table, not-// standalone views.-func getMaterializedViewNames(db *sql.DB, dbName string, dbType storepb.Engine) (map[string]bool, error) {+// The query is best-effort: an unsupported version logs and yields an empty set. Sync+// rollups (REFRESH_TYPE='ROLLUP') are excluded: they are rollup indexes on the base+// table, not standalone views.+func getMaterializedViewNames(db *sql.DB, dbName string) (map[string]bool, error) {names := make(map[string]bool)- var query string- switch dbType {- case storepb.Engine_DORIS:- query = fmt.Sprintf(`SELECT Name FROM mv_infos("database"="%s")`, dbName)- case storepb.Engine_STARROCKS:- // IFNULL guards the bare-string scan below: database/sql errors on a NULL->string scan.- query = fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, dbName)- default:- return names, nil- }+ // IFNULL guards the bare-string scan below: database/sql errors on a NULL->string scan.+ query := fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, dbName)rows, err := db.Query(query)if err != nil {@@ -307,18 +278,13 @@ func getMaterializedViewNames(db *sql.DB, dbName string, dbType storepb.Engine)defer rows.Close()for rows.Next() {- var name string- if dbType == storepb.Engine_STARROCKS {- var refreshType string- if err := rows.Scan(&name, &refreshType); err != nil {- return nil, err- }- if isSyncRollup(refreshType) {- continue- }- } else if err := rows.Scan(&name); err != nil {+ var name, refreshType string+ if err := rows.Scan(&name, &refreshType); err != nil {return nil, err}+ if isSyncRollup(refreshType) {+ continue+ }names[name] = true}if err := rows.Err(); err != nil {@@ -328,9 +294,9 @@ func getMaterializedViewNames(db *sql.DB, dbName string, dbType storepb.Engine)}// markMaterializedViews re-tags rows that are actually materialized views so the dump-// emits SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE' and-// StarRocks as 'VIEW'; either becomes 'MATERIALIZED VIEW' when its name is in the-// materialized-view set. Regular tables and views are left untouched.+// emits SHOW CREATE MATERIALIZED VIEW for them. StarRocks reports MVs as 'VIEW', which+// becomes 'MATERIALIZED VIEW' when the name is in the materialized-view set. Regular+// tables and views are left untouched.func markMaterializedViews(tables []*TableSchema, materializedViewNames map[string]bool) {for _, tbl := range tables {if !materializedViewNames[tbl.Name] {@@ -371,7 +337,7 @@ func getTableStmt(db *sql.DB, dbName, tblName, tblType string) (string, error) {createStmt = strings.TrimSuffix(createStmt, ";")return fmt.Sprintf(viewStmtFmt, tblName, createStmt), nilcase materializedViewType:- // For Doris materialized views, we use SHOW CREATE MATERIALIZED VIEW.+ // For materialized views we use SHOW CREATE MATERIALIZED VIEW.// This command returns 2 columns: materialized view name and create statement.query := fmt.Sprintf("SHOW CREATE MATERIALIZED VIEW `%s`.`%s`;", dbName, tblName)var createStmt, unused stringdiff --git a/backend/plugin/db/starrocks/dump_routine_probe_test.go b/backend/plugin/db/starrocks/dump_routine_probe_test.goindex 092f1cfac5..77af0603d1 100644--- a/backend/plugin/db/starrocks/dump_routine_probe_test.go+++ b/backend/plugin/db/starrocks/dump_routine_probe_test.go@@ -19,28 +19,23 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store")-// BYT-10211: neither Doris nor StarRocks has MySQL stored routines or events, and Doris-// kernels that dropped the PL/SQL grammar (SelectDB 4.0.12+/4.1.8+/5.0.5+, apache/doris#58700)-// reject `SHOW FUNCTION STATUS` outright, which aborted the whole schema sync. The dump must-// complete without ever sending the MySQL routine or event probes.+// BYT-10211: StarRocks has no MySQL stored routines or events, and a kernel that rejects+// `SHOW FUNCTION STATUS` outright aborted the whole schema sync. The dump must complete+// without ever sending the MySQL routine or event probes.func TestDumpSendsNoRoutineOrEventProbe(t *testing.T) {- for _, engine := range []storepb.Engine{storepb.Engine_DORIS, storepb.Engine_STARROCKS} {- t.Run(engine.String(), func(t *testing.T) {- dumpRecorder.reset(plsqlStrippedCatalog(engine))- sqlDB, err := sql.Open(dumpRecorderDriverName, "")- require.NoError(t, err)- defer sqlDB.Close()-- d := &Driver{dbType: engine, db: sqlDB, databaseName: "prod_tg"}- var out bytes.Buffer- require.NoError(t, d.Dump(context.Background(), &out, nil))-- require.Contains(t, out.String(), "CREATE TABLE `orders`")- require.Contains(t, out.String(), "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`")- for _, q := range dumpRecorder.recorded() {- require.NotRegexp(t, forbiddenProbe, q)- }- })+ dumpRecorder.reset(plsqlStrippedCatalog())+ sqlDB, err := sql.Open(dumpRecorderDriverName, "")+ require.NoError(t, err)+ defer sqlDB.Close()++ d := &Driver{dbType: storepb.Engine_STARROCKS, db: sqlDB, databaseName: "analytics"}+ var out bytes.Buffer+ require.NoError(t, d.Dump(context.Background(), &out, nil))++ require.Contains(t, out.String(), "CREATE TABLE `orders`")+ require.Contains(t, out.String(), "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`")+ for _, q := range dumpRecorder.recorded() {+ require.NotRegexp(t, forbiddenProbe, q)}}@@ -50,17 +45,15 @@ var ()// plsqlStrippedCatalog answers the dump's table and view queries with the column shapes a-// live Doris 4.1.3 returns (and the StarRocks materialized-view catalog in its own shape),-// and rejects the routine probes with the exact error a PL/SQL-stripped kernel returns.-func plsqlStrippedCatalog(engine storepb.Engine) func(string) ([]string, [][]driver.Value, error) {+// live StarRocks returns, and rejects the routine probes with the exact error a kernel+// without the routine grammar returns.+func plsqlStrippedCatalog() func(string) ([]string, [][]driver.Value, error) {return func(query string) ([]string, [][]driver.Value, error) {switch {case routineStatusProbe.MatchString(query):return nil, nil, &mysql.MySQLError{Number: 1105, SQLState: [5]byte{'H', 'Y', '0', '0', '0'}, Message: "errCode = 2, detailMessage = no viable alternative at input 'SHOW FUNCTION'(line 1, pos 5)"}case strings.Contains(query, "information_schema.TABLES"):return []string{"TABLE_NAME", "TABLE_TYPE"}, [][]driver.Value{{"orders", "BASE TABLE"}, {"v_orders", "VIEW"}}, nil- case strings.Contains(query, "mv_infos("):- return []string{"Name"}, nil, nilcase strings.Contains(query, "information_schema.materialized_views"):return []string{"TABLE_NAME", "REFRESH_TYPE"}, nil, nilcase strings.HasPrefix(query, "SHOW CREATE TABLE"):@@ -70,7 +63,7 @@ func plsqlStrippedCatalog(engine storepb.Engine) func(string) ([]string, [][]dricase strings.HasPrefix(query, "SHOW COLUMNS FROM"):return []string{"Field", "Type", "Null", "Key", "Default", "Extra"}, [][]driver.Value{{"id", "int", "YES", "", nil, ""}}, nil}- return nil, nil, errors.Errorf("unexpected statement for %s: %q", engine, query)+ return nil, nil, errors.Errorf("unexpected statement: %q", query)}}diff --git a/backend/plugin/db/starrocks/dump_test.go b/backend/plugin/db/starrocks/dump_test.goindex b38d5b6c9c..fab8a200ca 100644--- a/backend/plugin/db/starrocks/dump_test.go+++ b/backend/plugin/db/starrocks/dump_test.go@@ -14,17 +14,17 @@ func TestGetTemporaryView(t *testing.T) {}// BYT-9689: the dump path re-tags materialized-view rows so it emits-// SHOW CREATE MATERIALIZED VIEW for them. Doris reports MVs as 'BASE TABLE',-// StarRocks as 'VIEW'; both must become 'MATERIALIZED VIEW' when the name is a known-// MV, while regular tables and views are left untouched.+// SHOW CREATE MATERIALIZED VIEW for them. StarRocks reports MVs as 'VIEW', and some+// setups as 'BASE TABLE'; either must become 'MATERIALIZED VIEW' when the name is a+// known MV, while regular tables and views are left untouched.func TestMarkMaterializedViews(t *testing.T) {tables := []*TableSchema{- {Name: "mv_doris", TableType: baseTableType}, // Doris MV reported as BASE TABLE- {Name: "mv_starrocks", TableType: viewTableType}, // StarRocks MV reported as VIEW- {Name: "v_regular", TableType: viewTableType}, // regular view (not an MV)- {Name: "t_plain", TableType: baseTableType}, // plain table+ {Name: "mv_view", TableType: viewTableType}, // MV reported as VIEW+ {Name: "mv_base", TableType: baseTableType}, // MV reported as BASE TABLE+ {Name: "v_regular", TableType: viewTableType},+ {Name: "t_plain", TableType: baseTableType},}- markMaterializedViews(tables, map[string]bool{"mv_doris": true, "mv_starrocks": true})+ markMaterializedViews(tables, map[string]bool{"mv_view": true, "mv_base": true})require.Equal(t, materializedViewType, tables[0].TableType)require.Equal(t, materializedViewType, tables[1].TableType)diff --git a/backend/plugin/db/starrocks/starrocks.go b/backend/plugin/db/starrocks/starrocks.goindex 910b03b34a..645abfe2ed 100644--- a/backend/plugin/db/starrocks/starrocks.go+++ b/backend/plugin/db/starrocks/starrocks.go@@ -11,7 +11,7 @@ import ("strings""time"- mysqlomni "github.com/bytebase/omni/mysql/parser"+ starrocksomni "github.com/bytebase/omni/starrocks/parser""github.com/go-sql-driver/mysql""github.com/google/uuid""github.com/pkg/errors"@@ -28,9 +28,7 @@ import ("github.com/bytebase/bytebase/backend/plugin/db/util""github.com/bytebase/bytebase/backend/plugin/parser/base"- // Register how these engines plan a statement, for base.ExplainStatement below.- // This driver serves both.- _ "github.com/bytebase/bytebase/backend/plugin/parser/doris"+ // Register how StarRocks plans a statement, for base.ExplainStatement below._ "github.com/bytebase/bytebase/backend/plugin/parser/starrocks")@@ -44,10 +42,9 @@ var (func init() {db.Register(storepb.Engine_STARROCKS, newDriver)- db.Register(storepb.Engine_DORIS, newDriver)}-// Driver is the MySQL driver.+// Driver is the StarRocks driver.type Driver struct {dbType storepb.Enginedb *sql.DB@@ -62,7 +59,7 @@ func newDriver() db.Driver {return &Driver{}}-// Open opens a MySQL driver.+// Open opens a StarRocks driver.func (d *Driver) Open(_ context.Context, dbType storepb.Engine, connCfg db.ConnectionConfig) (db.Driver, error) {defer func() {for _, f := range d.openCleanUp {@@ -74,10 +71,7 @@ func (d *Driver) Open(_ context.Context, dbType storepb.Engine, connCfg db.Conneif strings.HasPrefix(connCfg.DataSource.Host, "/") {protocol = "unix"}- params := []string{"multiStatements=true", "maxAllowedPacket=0"}- if dbType == storepb.Engine_DORIS || dbType == storepb.Engine_STARROCKS {- params = append(params, "interpolateParams=true")- }+ params := []string{"multiStatements=true", "maxAllowedPacket=0", "interpolateParams=true"}if connCfg.DataSource.GetSshHost() != "" {sshClient, err := util.GetSSHClient(connCfg.DataSource)if err != nil {@@ -177,7 +171,7 @@ func (d *Driver) Execute(ctx context.Context, statement string, opts db.ExecuteO// Only preprocess when DELIMITER directives are present; normal// StarRocks SQL passes through unchanged.if containsDelimiterDirective(statement) {- segments := mysqlomni.Split(statement)+ segments := starrocksomni.Split(statement)var buf strings.Builderfor i, seg := range segments {if i > 0 {diff --git a/backend/plugin/db/starrocks/starrocks_test.go b/backend/plugin/db/starrocks/starrocks_test.goindex d84a8c1fdc..f6ef7b302f 100644--- a/backend/plugin/db/starrocks/starrocks_test.go+++ b/backend/plugin/db/starrocks/starrocks_test.go@@ -8,8 +8,7 @@ import (storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base"- // Register the doris + starrocks editor validators.- _ "github.com/bytebase/bytebase/backend/plugin/parser/doris"+ // Register the StarRocks editor validator._ "github.com/bytebase/bytebase/backend/plugin/parser/starrocks")@@ -72,19 +71,13 @@ func TestContainsDelimiterDirective(t *testing.T) {}}-// TestValidateSQLForEditor_EngineDispatch guards the #20562 P2 fix: QueryConn-// must split/validate via the connection's engine (d.dbType), not a hardcoded-// Engine_DORIS. On a StarRocks-only statement the two engines must disagree —-// doris rejects it (→ allQuery forced true, DDL wrongly routed through-// QueryContext), starrocks accepts it (→ correct allQuery=false).-func TestValidateSQLForEditor_EngineDispatch(t *testing.T) {- // StarRocks generated column AS (expr): doris wants GENERATED ALWAYS AS.- const ddl = "CREATE TABLE t (a INT, b INT AS (a + 1)) DUPLICATE KEY(a) DISTRIBUTED BY HASH(a)"-- _, srAllQuery, srErr := base.ValidateSQLForEditor(storepb.Engine_STARROCKS, ddl)- require.NoError(t, srErr, "starrocks parser should accept StarRocks DDL")- require.False(t, srAllQuery, "a CREATE TABLE is DDL, not a read-only query")-- _, _, dorisErr := base.ValidateSQLForEditor(storepb.Engine_DORIS, ddl)- require.Error(t, dorisErr, "doris parser rejects StarRocks AS(expr) DDL — why d.dbType matters")+// TestValidateSQLForEditorUsesStarRocksGrammar pins this driver to the StarRocks+// parser: a StarRocks-only spelling must be accepted, which routing the connection+// through a sibling engine's grammar would not do.+func TestValidateSQLForEditorUsesStarRocksGrammar(t *testing.T) {+ // StarRocks generated column AS (expr); Doris wants GENERATED ALWAYS AS.+ const starRocksOnlyDDL = "CREATE TABLE t (a INT, b INT AS (a + 1)) DUPLICATE KEY(a) DISTRIBUTED BY HASH(a)"+ _, allQuery, err := base.ValidateSQLForEditor(storepb.Engine_STARROCKS, starRocksOnlyDDL)+ require.NoError(t, err)+ require.False(t, allQuery, "a CREATE TABLE is DDL, not a read-only query")}diff --git a/backend/plugin/db/starrocks/sync.go b/backend/plugin/db/starrocks/sync.goindex 510f27bc8b..dd9cbbd128 100644--- a/backend/plugin/db/starrocks/sync.go+++ b/backend/plugin/db/starrocks/sync.go@@ -24,15 +24,10 @@ const ()// systemDatabaseExclusion returns the quoted, comma-joined system databases to-// exclude from instance discovery, per the driver's engine. StarRocks adds its-// read-only `sys` metadatabase (Doris has no sys), matching the parser-side-// system-database classification.-func (d *Driver) systemDatabaseExclusion() string {- dbs := []string{"'information_schema'", "'_statistics_'"}- if d.dbType == storepb.Engine_STARROCKS {- dbs = append(dbs, "'sys'")- }- return strings.Join(dbs, ", ")+// exclude from instance discovery, matching the parser-side system-database+// classification. `sys` is StarRocks's read-only metadatabase, not a user database.+func systemDatabaseExclusion() string {+ return strings.Join([]string{"'information_schema'", "'_statistics_'", "'sys'"}, ", ")}// SyncInstance syncs the instance.@@ -56,7 +51,7 @@ func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error)}// Query db info- where := fmt.Sprintf("LOWER(SCHEMA_NAME) NOT IN (%s)", d.systemDatabaseExclusion())+ where := fmt.Sprintf("LOWER(SCHEMA_NAME) NOT IN (%s)", systemDatabaseExclusion())query := `SELECTSCHEMA_NAME,@@ -80,10 +75,8 @@ func (d *Driver) SyncInstance(ctx context.Context) (*db.InstanceMetadata, error)); err != nil {return nil, err}- if d.dbType == storepb.Engine_DORIS || d.dbType == storepb.Engine_STARROCKS {- database.CharacterSet = ""- database.Collation = ""- }+ database.CharacterSet = ""+ database.Collation = ""databases = append(databases, database)}if err := rows.Err(); err != nil {@@ -127,9 +120,8 @@ func isSyncRollup(refreshType string) bool {// isMaterializedView reports whether a table row is a materialized view, based on its// table type and membership in the materialized-view set. StarRocks reports-// materialized views with TABLE_TYPE='VIEW', Doris with 'BASE TABLE', and some setups-// with 'MATERIALIZED VIEW'; in every case, presence in materializedViewMap is the-// authoritative signal.+// materialized views with TABLE_TYPE='VIEW', and some setups with 'MATERIALIZED VIEW';+// in either case, presence in materializedViewMap is the authoritative signal.func isMaterializedView(tableType string, key db.TableKey, materializedViewMap map[db.TableKey]*metadatapb.MaterializedViewMetadata) bool {if _, ok := materializedViewMap[key]; !ok {return false@@ -143,26 +135,15 @@ func isMaterializedView(tableType string, key db.TableKey, materializedViewMap m}// getMaterializedViews returns the materialized views of the current database keyed by-// table name. Doris and StarRocks expose them through different catalogs, so the query-// and projection are engine-specific. The query is best-effort: an engine/version that-// does not support it is logged and yields an empty set rather than failing the sync.+// table name. The query is best-effort: a version that does not support it is logged+// and yields an empty set rather than failing the sync.func (d *Driver) getMaterializedViews(ctx context.Context) (map[db.TableKey]*metadatapb.MaterializedViewMetadata, error) {materializedViewMap := make(map[db.TableKey]*metadatapb.MaterializedViewMetadata)- var query string- switch d.dbType {- case storepb.Engine_DORIS:- // Doris exposes materialized views via the mv_infos() table-valued function.- // https://doris.apache.org/docs/sql-manual/sql-functions/table-valued-functions/mv_infos- query = fmt.Sprintf(`SELECT Name, QuerySql FROM mv_infos("database"="%s")`, d.databaseName)- case storepb.Engine_STARROCKS:- // StarRocks exposes materialized views via information_schema.materialized_views.- // REFRESH_TYPE separates async MVs from synchronous rollups, which are excluded.- // IFNULL guards the bare-string scans below: database/sql errors on a NULL->string scan.- query = fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, ''), IFNULL(MATERIALIZED_VIEW_DEFINITION, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, d.databaseName)- default:- return materializedViewMap, nil- }+ // StarRocks exposes materialized views via information_schema.materialized_views.+ // REFRESH_TYPE separates async MVs from synchronous rollups, which are excluded.+ // IFNULL guards the bare-string scans below: database/sql errors on a NULL->string scan.+ query := fmt.Sprintf(`SELECT TABLE_NAME, IFNULL(REFRESH_TYPE, ''), IFNULL(MATERIALIZED_VIEW_DEFINITION, '') FROM information_schema.materialized_views WHERE TABLE_SCHEMA = '%s'`, d.databaseName)rows, err := d.db.QueryContext(ctx, query)if err != nil {@@ -174,17 +155,13 @@ func (d *Driver) getMaterializedViews(ctx context.Context) (map[db.TableKey]*metfor rows.Next() {materializedView := &metadatapb.MaterializedViewMetadata{}- if d.dbType == storepb.Engine_STARROCKS {- var refreshType string- if err := rows.Scan(&materializedView.Name, &refreshType, &materializedView.Definition); err != nil {- return nil, err- }- if isSyncRollup(refreshType) {- continue- }- } else if err := rows.Scan(&materializedView.Name, &materializedView.Definition); err != nil {+ var refreshType string+ if err := rows.Scan(&materializedView.Name, &refreshType, &materializedView.Definition); err != nil {return nil, err}+ if isSyncRollup(refreshType) {+ continue+ }key := db.TableKey{Schema: "", Table: materializedView.Name}materializedViewMap[key] = materializedView}@@ -354,7 +331,7 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe}key := db.TableKey{Schema: "", Table: tableName}- // StarRocks reports materialized views as TABLE_TYPE='VIEW', Doris as 'BASE TABLE';+ // StarRocks reports materialized views as TABLE_TYPE='VIEW';// route any row whose name is in the materialized-view set to MaterializedViews.if isMaterializedView(tableType, key, materializedViewMap) {materializedView := materializedViewMap[key]@@ -418,10 +395,8 @@ func (d *Driver) SyncDBSchema(ctx context.Context) (*metadatapb.DatabaseSchemaMe}// "characterSet":"utf8\u0000", "collation":"utf8_general_ci\u0000".// ERROR: unsupported Unicode escape sequence (SQLSTATE 22P05).- if d.dbType == storepb.Engine_DORIS || d.dbType == storepb.Engine_STARROCKS {- databaseMetadata.CharacterSet = ""- databaseMetadata.Collation = ""- }+ databaseMetadata.CharacterSet = ""+ databaseMetadata.Collation = ""return databaseMetadata, err}diff --git a/backend/plugin/db/starrocks/sync_test.go b/backend/plugin/db/starrocks/sync_test.goindex 8a823b1e0d..ef4a00df64 100644--- a/backend/plugin/db/starrocks/sync_test.go+++ b/backend/plugin/db/starrocks/sync_test.go@@ -6,25 +6,22 @@ import (metadatapb "github.com/bytebase/omni/metadata""github.com/stretchr/testify/require"- storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/db")// BYT-9689: StarRocks reports materialized views in information_schema.tables with// TABLE_TYPE='VIEW' and omits them from information_schema.views (verified on a live-// 3.4.10 BE-alive smoke test), while Doris reports them as 'BASE TABLE'. The shared-// driver previously only matched Doris's 'BASE TABLE' rows, so StarRocks MVs were-// dropped. Membership in the materialized-view set is the authoritative signal.+// 3.4.10 BE-alive smoke test), so the table type alone cannot identify one. Membership+// in the materialized-view set is the authoritative signal, whatever type the row carries.func TestIsMaterializedView(t *testing.T) {mvMap := map[db.TableKey]*metadatapb.MaterializedViewMetadata{{Schema: "", Table: "mv_async"}: {Name: "mv_async"},}mvKey := db.TableKey{Schema: "", Table: "mv_async"}- // StarRocks reports the MV as VIEW; Doris as BASE TABLE; both map to an MV.require.True(t, isMaterializedView(viewTableType, mvKey, mvMap))- require.True(t, isMaterializedView(baseTableType, mvKey, mvMap))require.True(t, isMaterializedView(materializedViewType, mvKey, mvMap))+ require.True(t, isMaterializedView(baseTableType, mvKey, mvMap))// A name absent from the MV set is never a materialized view.require.False(t, isMaterializedView(viewTableType, db.TableKey{Table: "v_regular"}, mvMap))@@ -45,17 +42,12 @@ func TestIsSyncRollup(t *testing.T) {require.False(t, isSyncRollup("")) // IFNULL-coalesced NULL or a genuinely empty type.}-// #20566 review (P2): SyncInstance's system-database exclusion must match the-// parser's per-engine system set — StarRocks's read-only `sys` metadatabase-// must be excluded so it isn't imported as a user database. Doris has no `sys`,-// so excluding it there would risk hiding a user db named sys (engine-aware).+// #20566 review (P2): SyncInstance's system-database exclusion must match the parser's+// system set. `sys` is StarRocks's read-only metadatabase and must be excluded so it+// isn't imported as a user database.func TestSystemDatabaseExclusion(t *testing.T) {- sr := (&Driver{dbType: storepb.Engine_STARROCKS}).systemDatabaseExclusion()- require.Contains(t, sr, "'sys'", "StarRocks sync must exclude the sys metadatabase")- require.Contains(t, sr, "'information_schema'")- require.Contains(t, sr, "'_statistics_'")-- doris := (&Driver{dbType: storepb.Engine_DORIS}).systemDatabaseExclusion()- require.NotContains(t, doris, "'sys'", "Doris must not exclude sys (StarRocks-only)")- require.Contains(t, doris, "'information_schema'")+ got := systemDatabaseExclusion()+ require.Contains(t, got, "'information_schema'")+ require.Contains(t, got, "'_statistics_'")+ require.Contains(t, got, "'sys'")}diff --git a/backend/plugin/parser/doris/doris.go b/backend/plugin/parser/doris/doris.goindex 7bd674f1f6..b13b78b0c9 100644--- a/backend/plugin/parser/doris/doris.go+++ b/backend/plugin/parser/doris/doris.go@@ -19,6 +19,7 @@ func init() {// omniAST wraps an omni AST node to implement the base.AST interface.type omniAST struct {node ast.Node+ text stringstartPos *storepb.Position}@@ -67,6 +68,7 @@ func parseDorisStatements(statement string) ([]base.ParsedStatement, error) {}ps.AST = &omniAST{node: node,+ text: stmt.Text,startPos: stmt.Start,}result = append(result, ps)@@ -103,6 +105,7 @@ func parseDorisSQL(statement string) ([]*omniAST, error) {}result = append(result, &omniAST{node: node,+ text: stmt.Text,startPos: stmt.Start,})}diff --git a/backend/plugin/parser/doris/resource_change.go b/backend/plugin/parser/doris/resource_change.gonew file mode 100644index 0000000000..a166bd1d5d--- /dev/null+++ b/backend/plugin/parser/doris/resource_change.go@@ -0,0 +1,161 @@+package doris++import (+ "strings"++ "github.com/bytebase/omni/doris/ast"+ "github.com/pkg/errors"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model"+)++func init() {+ base.RegisterExtractChangedResourcesFunc(storepb.Engine_DORIS, extractChangedResources)+}++func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {+ changedResources := model.NewChangedResources(dbMetadata)++ var dmlCount, insertCount int+ var dmlStatements []string++ addTable := func(name *ast.ObjectName, affectsExistingRows bool) {+ db, table := objectName(name, currentDatabase)+ if table == "" {+ return+ }+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, affectsExistingRows)+ }++ // addObjectDatabase records a database-only write target for a non-table object DDL+ // (view, materialized view, routine) ONLY when the object's own name carries an+ // explicit database qualifier. An unqualified name records nothing, leaving the+ // request database to stand in. SUP-222.+ addObjectDatabase := func(name *ast.ObjectName) {+ if db, _ := objectName(name, ""); db != "" {+ changedResources.AddDatabase(db)+ }+ }++ for _, unifiedAST := range asts {+ parsed, ok := unifiedAST.(*omniAST)+ if !ok {+ return nil, errors.New("expected omniAST for Doris")+ }+ if parsed.node == nil {+ continue+ }++ switch n := parsed.node.(type) {+ case *ast.CreateTableStmt:+ addTable(n.Name, false)++ case *ast.DropTableStmt:+ addTable(n.Name, true)++ case *ast.TruncateTableStmt:+ addTable(n.Target, true)++ case *ast.AlterTableStmt:+ addTable(n.Name, true)++ case *ast.CreateIndexStmt:+ addTable(n.Table, false)++ case *ast.DropIndexStmt:+ addTable(n.Table, false)++ case *ast.LoadDataStmt:+ // One LOAD LABEL carries a data description per target table.+ for _, desc := range n.DataDescs {+ if desc != nil {+ addTable(desc.Target, false)+ }+ }++ case *ast.InsertStmt:+ addTable(n.Target, false)+ if len(n.Values) > 0 {+ insertCount += len(n.Values)+ continue+ }+ dmlCount+++ dmlStatements = append(dmlStatements, statementText(parsed))++ case *ast.UpdateStmt:+ // Doris updates a single target; FROM names read sources, not write targets.+ addTable(n.Target, false)+ dmlCount+++ dmlStatements = append(dmlStatements, statementText(parsed))++ case *ast.DeleteStmt:+ // USING names read sources, as FROM does for UPDATE.+ addTable(n.Target, false)+ dmlCount+++ dmlStatements = append(dmlStatements, statementText(parsed))++ case *ast.MergeStmt:+ addTable(n.Target, false)+ dmlCount+++ dmlStatements = append(dmlStatements, statementText(parsed))++ // Non-table object DDL: gate by the object's own explicit database qualifier only.+ // CREATE FUNCTION is absent because its AST keeps the name inside RawSignature+ // rather than as an ObjectName, so there is nothing to qualify by.+ case *ast.CreateViewStmt:+ addObjectDatabase(n.Name)+ case *ast.AlterViewStmt:+ addObjectDatabase(n.Name)+ case *ast.DropViewStmt:+ addObjectDatabase(n.Name)+ case *ast.CreateMTMVStmt:+ addObjectDatabase(n.Name)+ case *ast.AlterMTMVStmt:+ addObjectDatabase(n.Name)+ case *ast.DropMTMVStmt:+ addObjectDatabase(n.Name)+ case *ast.DropFunctionStmt:+ addObjectDatabase(n.Name)+ case *ast.CreateProcedureStmt:+ addObjectDatabase(n.Name)+ case *ast.DropProcedureStmt:+ addObjectDatabase(n.Name)++ default:+ }+ }++ return &base.ChangeSummary{+ ChangedResources: changedResources,+ DMLStatements: dmlStatements,+ DMLCount: dmlCount,+ InsertCount: insertCount,+ }, nil+}++// objectName splits an ObjectName into its database and object name. Doris writes a+// name as table, db.table or catalog.db.table, so the object is the last part and the+// database the one before it; a catalog qualifier is dropped, since changed resources+// are tracked per database.+func objectName(name *ast.ObjectName, defaultDatabase string) (string, string) {+ if name == nil || len(name.Parts) == 0 {+ return defaultDatabase, ""+ }+ parts := name.Parts+ database := defaultDatabase+ if len(parts) >= 2 {+ database = parts[len(parts)-2]+ }+ return database, parts[len(parts)-1]+}++// statementText returns the text of a statement, ensuring it ends with a semicolon.+func statementText(a *omniAST) string {+ text := strings.TrimSpace(a.text)+ if !strings.HasSuffix(text, ";") {+ text += ";"+ }+ return text+}diff --git a/backend/plugin/parser/doris/resource_change_test.go b/backend/plugin/parser/doris/resource_change_test.gonew file mode 100644index 0000000000..8383ccd8d8--- /dev/null+++ b/backend/plugin/parser/doris/resource_change_test.go@@ -0,0 +1,149 @@+package doris++import (+ "testing"++ "github.com/stretchr/testify/require"++ metadatapb "github.com/bytebase/omni/metadata"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model"+)++// changedTables flattens the built resources into "database.table" strings.+func changedTables(t *testing.T, statement string) []string {+ t.Helper()+ summary := extract(t, statement, nil)+ var result []string+ for _, database := range summary.ChangedResources.Build().GetDatabases() {+ for _, schema := range database.GetSchemas() {+ for _, table := range schema.GetTables() {+ result = append(result, database.GetName()+"."+table.GetName())+ }+ }+ }+ return result+}++func extract(t *testing.T, statement string, dbMetadata *model.DatabaseMetadata) *base.ChangeSummary {+ t.Helper()+ stmts, err := base.ParseStatements(storepb.Engine_DORIS, statement)+ require.NoError(t, err)+ summary, err := base.ExtractChangedResources(storepb.Engine_DORIS, "db", "", dbMetadata, base.ExtractASTs(stmts), statement)+ require.NoError(t, err)+ return summary+}++func TestExtractChangedResourcesTables(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ want []string+ }{+ {"create table takes the request database", "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"db.t"}},+ {"a qualified name keeps its own database", "CREATE TABLE other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"other.t"}},+ {"a catalog qualifier is dropped", "CREATE TABLE cat.other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"other.t"}},+ {"drop table", "DROP TABLE t;", []string{"db.t"}},+ {"truncate table", "TRUNCATE TABLE t;", []string{"db.t"}},+ {"alter table", "ALTER TABLE t ADD COLUMN b INT;", []string{"db.t"}},+ {"create index names its table", "CREATE INDEX idx ON t (a);", []string{"db.t"}},+ {"drop index names its table", "DROP INDEX idx ON t;", []string{"db.t"}},+ {"insert names its target", "INSERT INTO t VALUES (1);", []string{"db.t"}},+ {"update names its target, not its source", "UPDATE t SET a = 1 WHERE a IN (SELECT a FROM s);", []string{"db.t"}},+ {"delete names its target, not its source", "DELETE FROM t WHERE a = 1;", []string{"db.t"}},+ {"a view records no table", "CREATE VIEW v AS SELECT 1;", nil},+ } {+ t.Run(tc.name, func(t *testing.T) {+ require.ElementsMatch(t, tc.want, changedTables(t, tc.statement))+ })+ }+}++func TestExtractChangedResourcesDMLAccounting(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ wantDML int+ wantInserts int+ wantTexts []string+ }{+ {name: "insert values counts rows, not statements", statement: "INSERT INTO t VALUES (1), (2), (3);", wantInserts: 3},+ {name: "insert select is a DML statement", statement: "INSERT INTO t SELECT a FROM s;", wantDML: 1, wantTexts: []string{"INSERT INTO t SELECT a FROM s;"}},+ {name: "update is a DML statement", statement: "UPDATE t SET a = 1 WHERE a = 2;", wantDML: 1, wantTexts: []string{"UPDATE t SET a = 1 WHERE a = 2;"}},+ {name: "delete is a DML statement", statement: "DELETE FROM t WHERE a = 1;", wantDML: 1, wantTexts: []string{"DELETE FROM t WHERE a = 1;"}},+ {name: "DDL is neither", statement: "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;"},+ {+ name: "a script accumulates both",+ statement: "INSERT INTO t VALUES (1), (2);\nUPDATE t SET a = 1;\nDELETE FROM t;",+ wantDML: 2,+ wantInserts: 2,+ wantTexts: []string{"UPDATE t SET a = 1;", "DELETE FROM t;"},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ summary := extract(t, tc.statement, nil)+ require.Equal(t, tc.wantDML, summary.DMLCount)+ require.Equal(t, tc.wantInserts, summary.InsertCount)+ require.Equal(t, tc.wantTexts, summary.DMLStatements)+ })+ }+}++func TestExtractChangedResourcesDatabaseOnlyTargets(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ want []string+ }{+ {"a qualified view records its database", "CREATE VIEW other.v AS SELECT 1;", []string{"other"}},+ {"an unqualified view records nothing", "CREATE VIEW v AS SELECT 1;", []string{}},+ {"alter view", "ALTER VIEW other.v AS SELECT 1;", []string{"other"}},+ {"drop view", "DROP VIEW other.v;", []string{"other"}},+ {"a qualified materialized view records its database", "DROP MATERIALIZED VIEW other.mv;", []string{"other"}},+ {"drop procedure", "DROP PROCEDURE other.p;", []string{"other"}},+ {"a table DDL records no database-only target", "CREATE TABLE other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ summary := extract(t, tc.statement, nil)+ require.Equal(t, tc.want, summary.ChangedResources.GetDatabaseOnlyTargets())+ })+ }+}++func TestExtractChangedResourcesAffectedRows(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{Tables: []*metadatapb.TableMetadata{{Name: "t", RowCount: 42}}}},+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_DORIS, false /* caseSensitive */)++ for _, tc := range []struct {+ name string+ statement string+ want int64+ }{+ {"drop discards existing rows", "DROP TABLE t;", 42},+ {"truncate discards existing rows", "TRUNCATE TABLE t;", 42},+ {"alter rewrites existing rows", "ALTER TABLE t ADD COLUMN b INT;", 42},+ {"create has no existing rows", "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", 0},+ {"insert leaves existing rows alone", "INSERT INTO t VALUES (1);", 0},+ {"create index leaves existing rows alone", "CREATE INDEX idx ON t (a);", 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ summary := extract(t, tc.statement, dbMetadata)+ require.Equal(t, tc.want, summary.ChangedResources.CountAffectedTableRows())+ })+ }+}++// foreignAST stands in for another engine's AST, which reached this extractor+// through the shared registry before Doris had one of its own.+type foreignAST struct{}++func (*foreignAST) ASTStartPosition() *storepb.Position { return nil }++func TestExtractChangedResourcesRejectsForeignAST(t *testing.T) {+ _, err := extractChangedResources("db", "", nil, []base.AST{&foreignAST{}}, "")+ require.ErrorContains(t, err, "expected omniAST for Doris")+}diff --git a/backend/plugin/parser/mysql/query_span_extractor.go b/backend/plugin/parser/mysql/query_span_extractor.goindex ca4a6a1326..da9f4579dc 100644--- a/backend/plugin/parser/mysql/query_span_extractor.go+++ b/backend/plugin/parser/mysql/query_span_extractor.go@@ -10,7 +10,6 @@ import ("github.com/bytebase/omni/mysql/ast""github.com/pkg/errors"- storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/parser/base""github.com/bytebase/bytebase/backend/store/model")@@ -162,7 +161,6 @@ func (q *querySpanExtractor) getAllTableColumnSources(databaseName, tableName st// SELECT first, by the rules of the clause the reference is in, then each// enclosing SELECT outward, its tables before the select list it exposes.func (q *querySpanExtractor) getFieldColumnSource(databaseName, tableName, fieldName string) (base.SourceColumnSet, error) {- databaseName = q.filterClusterName(databaseName)if sourceColumnSet, ok := q.resolveInCurrentSelect(databaseName, tableName, fieldName); ok {return sourceColumnSet, nil}@@ -229,16 +227,6 @@ func (q *querySpanExtractor) findColumnInTables(tables []base.TableSource, databreturn nil, false}-func (q *querySpanExtractor) filterClusterName(databaseName string) string {- if q.gCtx.Engine == storepb.Engine_STARROCKS {- list := strings.Split(databaseName, ":")- if len(list) > 1 {- databaseName = list[len(list)-1]- }- }- return databaseName-}-func (q *querySpanExtractor) findTableSchema(databaseName, tableName string) (base.TableSource, error) {if databaseName == "" {for i := len(q.ctes) - 1; i >= 0; i-- {@@ -253,7 +241,6 @@ func (q *querySpanExtractor) findTableSchema(databaseName, tableName string) (baif databaseName == "" {databaseName = q.defaultDatabase}- databaseName = q.filterClusterName(databaseName)var dbMetadata *model.DatabaseMetadataallDatabaseNames, err := q.gCtx.ListDatabaseNamesFunc(q.ctx, q.gCtx.InstanceID)@@ -456,7 +443,7 @@ func (q *omniQuerySpanExtractor) getOmniQuerySpan(ctx context.Context, statement}root := stmts.Items[0]- accessTables := collectOmniAccessTables(root, q.defaultDatabase, q.gCtx.Engine == storepb.Engine_STARROCKS)+ accessTables := collectOmniAccessTables(root, q.defaultDatabase)allSystems, mixed := isMixedQuery(accessTables, q.ignoreCaseSensitive)if mixed {return nil, base.MixUserSystemTablesError@@ -1635,13 +1622,13 @@ func isOmniDescribeStatement(source string, loc ast.Loc) bool {return len(fields) > 0 && (fields[0] == "DESCRIBE" || fields[0] == "DESC")}-func collectOmniAccessTables(root ast.Node, defaultDatabase string, normalizeStarRocksCluster bool) base.SourceColumnSet {+func collectOmniAccessTables(root ast.Node, defaultDatabase string) base.SourceColumnSet {result := make(base.SourceColumnSet)- collectOmniAccessTablesFromNode(result, root, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, root, defaultDatabase)return result}-func collectOmniAccessTablesFromNode(result base.SourceColumnSet, node ast.Node, defaultDatabase string, normalizeStarRocksCluster bool) {+func collectOmniAccessTablesFromNode(result base.SourceColumnSet, node ast.Node, defaultDatabase string) {switch n := node.(type) {case nil:case *ast.SelectStmt:@@ -1650,206 +1637,206 @@ func collectOmniAccessTablesFromNode(result base.SourceColumnSet, node ast.Node,}for _, cte := range n.CTEs {if cte != nil {- collectOmniAccessTablesFromNode(result, cte.Select, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, cte.Select, defaultDatabase)}}if n.Left != nil {- collectOmniAccessTablesFromNode(result, n.Left, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, n.Left, defaultDatabase)}if n.Right != nil {- collectOmniAccessTablesFromNode(result, n.Right, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, n.Right, defaultDatabase)}if inner := n.ParenSource; inner != nil {- collectOmniAccessTablesFromNode(result, inner, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, inner, defaultDatabase)}if tableStmt := n.TableSource; tableStmt != nil {- collectOmniAccessTablesFromNode(result, tableStmt, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, tableStmt, defaultDatabase)}if valuesStmt := n.ValuesSource; valuesStmt != nil {- collectOmniAccessTablesFromNode(result, valuesStmt, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, valuesStmt, defaultDatabase)}for _, target := range n.TargetList {- collectOmniAccessTablesFromExpr(result, target, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, target, defaultDatabase)}for _, tableExpr := range n.From {- collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase)}- collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase)for _, groupBy := range n.GroupBy {- collectOmniAccessTablesFromExpr(result, groupBy, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, groupBy, defaultDatabase)}- collectOmniAccessTablesFromExpr(result, n.Having, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, n.Having, defaultDatabase)for _, orderBy := range n.OrderBy {if orderBy != nil {- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)}}case *ast.ExplainStmt:- collectOmniAccessTablesFromNode(result, n.Stmt, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, n.Stmt, defaultDatabase)case *ast.InsertStmt:- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)for _, row := range n.Values {for _, expr := range row {- collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase)}}- collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromNode(result, n.TableSource, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase)+ collectOmniAccessTablesFromNode(result, n.TableSource, defaultDatabase)for _, assignment := range n.SetList {if assignment != nil {- collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase)}}for _, assignment := range n.OnDuplicateKey {if assignment != nil {- collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase)}}case *ast.UpdateStmt:for _, tableExpr := range n.Tables {- collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase)}for _, assignment := range n.SetList {if assignment != nil {- collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase)}}- collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase)for _, orderBy := range n.OrderBy {if orderBy != nil {- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)}}case *ast.DeleteStmt:for _, tableExpr := range n.Tables {- collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase)}for _, tableExpr := range n.Using {- collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableExpr(result, tableExpr, defaultDatabase)}- collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase)for _, orderBy := range n.OrderBy {if orderBy != nil {- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)}}case *ast.CreateTableStmt:- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromTableRef(result, n.Like, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)+ collectOmniAccessTablesFromTableRef(result, n.Like, defaultDatabase)+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase)case *ast.CreateIndexStmt:- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)for _, column := range n.Columns {if column != nil {- collectOmniAccessTablesFromExpr(result, column.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, column.Expr, defaultDatabase)}}case *ast.CreateViewStmt:- collectOmniAccessTablesFromTableRef(result, n.Name, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Name, defaultDatabase)+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase)case *ast.AlterTableStmt:- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)for _, command := range n.Commands {if command == nil {continue}- collectOmniAccessTablesFromExpr(result, command.DefaultExpr, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromTableRef(result, command.ExchangeTable, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromOrderBy(result, command.OrderByItems, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, command.DefaultExpr, defaultDatabase)+ collectOmniAccessTablesFromTableRef(result, command.ExchangeTable, defaultDatabase)+ collectOmniAccessTablesFromOrderBy(result, command.OrderByItems, defaultDatabase)}case *ast.DropTableStmt:- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)case *ast.DropIndexStmt:- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)case *ast.DropViewStmt:- collectOmniAccessTablesFromTableRefs(result, n.Views, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRefs(result, n.Views, defaultDatabase)case *ast.RenameTableStmt:for _, pair := range n.Pairs {if pair == nil {continue}- collectOmniAccessTablesFromTableRef(result, pair.Old, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromTableRef(result, pair.New, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, pair.Old, defaultDatabase)+ collectOmniAccessTablesFromTableRef(result, pair.New, defaultDatabase)}case *ast.TruncateStmt:- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)case *ast.AnalyzeTableStmt:- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)case *ast.OptimizeTableStmt:- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)case *ast.CheckTableStmt:- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)case *ast.RepairTableStmt:- collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRefs(result, n.Tables, defaultDatabase)case *ast.TableStmt:if n == nil {return}- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)+ collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase)+ collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase)case *ast.ValuesStmt:for _, row := range n.Rows {for _, expr := range row {- collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase)}}- collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromOrderBy(result, n.OrderBy, defaultDatabase)+ collectOmniAccessTablesFromLimit(result, n.Limit, defaultDatabase)case *ast.CallStmt:for _, arg := range n.Args {- collectOmniAccessTablesFromExpr(result, arg, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, arg, defaultDatabase)}case *ast.SetStmt:for _, assignment := range n.Assignments {if assignment != nil {- collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, assignment.Value, defaultDatabase)}}case *ast.DoStmt:for _, expr := range n.Exprs {- collectOmniAccessTablesFromExpr(result, expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, expr, defaultDatabase)}case *ast.HandlerOpenStmt:- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)case *ast.HandlerReadStmt:- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, n.Where, defaultDatabase)case *ast.HandlerCloseStmt:- collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n.Table, defaultDatabase)case *ast.LockTablesStmt:for _, table := range n.Tables {if table != nil {- collectOmniAccessTablesFromTableRef(result, table.Table, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, table.Table, defaultDatabase)}}default:}}-func collectOmniAccessTablesFromOrderBy(result base.SourceColumnSet, orderByItems []*ast.OrderByItem, defaultDatabase string, normalizeStarRocksCluster bool) {+func collectOmniAccessTablesFromOrderBy(result base.SourceColumnSet, orderByItems []*ast.OrderByItem, defaultDatabase string) {for _, orderBy := range orderByItems {if orderBy != nil {- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)}}}-func collectOmniAccessTablesFromLimit(result base.SourceColumnSet, limit *ast.Limit, defaultDatabase string, normalizeStarRocksCluster bool) {+func collectOmniAccessTablesFromLimit(result base.SourceColumnSet, limit *ast.Limit, defaultDatabase string) {if limit == nil {return}- collectOmniAccessTablesFromExpr(result, limit.Count, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, limit.Offset, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, limit.Count, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, limit.Offset, defaultDatabase)}-func collectOmniAccessTablesFromTableRefs(result base.SourceColumnSet, tableRefs []*ast.TableRef, defaultDatabase string, normalizeStarRocksCluster bool) {+func collectOmniAccessTablesFromTableRefs(result base.SourceColumnSet, tableRefs []*ast.TableRef, defaultDatabase string) {for _, tableRef := range tableRefs {- collectOmniAccessTablesFromTableRef(result, tableRef, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, tableRef, defaultDatabase)}}-func collectOmniAccessTablesFromTableRef(result base.SourceColumnSet, tableRef *ast.TableRef, defaultDatabase string, normalizeStarRocksCluster bool) {+func collectOmniAccessTablesFromTableRef(result base.SourceColumnSet, tableRef *ast.TableRef, defaultDatabase string) {if tableRef == nil {return}@@ -1859,8 +1846,6 @@ func collectOmniAccessTablesFromTableRef(result base.SourceColumnSet, tableRef *database := tableRef.Schemaif database == "" {database = defaultDatabase- } else if normalizeStarRocksCluster {- database = filterStarRocksClusterName(database)}result[base.ColumnResource{Database: database,@@ -1872,121 +1857,113 @@ func isOmniDualTable(tableRef *ast.TableRef) bool {return tableRef != nil && tableRef.Schema == "" && strings.EqualFold(tableRef.Name, "dual")}-func collectOmniAccessTablesFromTableExpr(result base.SourceColumnSet, tableExpr ast.TableExpr, defaultDatabase string, normalizeStarRocksCluster bool) {+func collectOmniAccessTablesFromTableExpr(result base.SourceColumnSet, tableExpr ast.TableExpr, defaultDatabase string) {switch n := tableExpr.(type) {case *ast.TableRef:- collectOmniAccessTablesFromTableRef(result, n, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableRef(result, n, defaultDatabase)case *ast.JoinClause:- collectOmniAccessTablesFromTableExpr(result, n.Left, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromTableExpr(result, n.Right, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromTableExpr(result, n.Left, defaultDatabase)+ collectOmniAccessTablesFromTableExpr(result, n.Right, defaultDatabase)switch condition := n.Condition.(type) {case *ast.OnCondition:- collectOmniAccessTablesFromExpr(result, condition.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, condition.Expr, defaultDatabase)default:}case *ast.SubqueryExpr:- collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, n.Select, defaultDatabase)case *ast.JsonTableExpr:- collectOmniAccessTablesFromExpr(result, n.Expr, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, n.Path, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, n.Expr, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, n.Path, defaultDatabase)default:}}-func collectOmniAccessTablesFromExpr(result base.SourceColumnSet, expr ast.ExprNode, defaultDatabase string, normalizeStarRocksCluster bool) {+func collectOmniAccessTablesFromExpr(result base.SourceColumnSet, expr ast.ExprNode, defaultDatabase string) {switch e := expr.(type) {case nil:case *ast.ResTarget:- collectOmniAccessTablesFromExpr(result, e.Val, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Val, defaultDatabase)case *ast.BinaryExpr:- collectOmniAccessTablesFromExpr(result, e.Left, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, e.Right, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Left, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, e.Right, defaultDatabase)case *ast.UnaryExpr:- collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase)case *ast.FuncCallExpr:for _, arg := range e.Args {- collectOmniAccessTablesFromExpr(result, arg, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, arg, defaultDatabase)}- collectOmniAccessTablesFromExpr(result, e.Separator, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Separator, defaultDatabase)for _, orderBy := range e.OrderBy {if orderBy != nil {- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)}}if e.Over != nil {for _, partitionBy := range e.Over.PartitionBy {- collectOmniAccessTablesFromExpr(result, partitionBy, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, partitionBy, defaultDatabase)}for _, orderBy := range e.Over.OrderBy {if orderBy != nil {- collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, orderBy.Expr, defaultDatabase)}}}case *ast.SubqueryExpr:- collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase)case *ast.CaseExpr:- collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Operand, defaultDatabase)for _, when := range e.Whens {if when == nil {continue}- collectOmniAccessTablesFromExpr(result, when.Cond, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, when.Result, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, when.Cond, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, when.Result, defaultDatabase)}- collectOmniAccessTablesFromExpr(result, e.Default, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Default, defaultDatabase)case *ast.BetweenExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, e.Low, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, e.High, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, e.Low, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, e.High, defaultDatabase)case *ast.InExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)for _, item := range e.List {- collectOmniAccessTablesFromExpr(result, item, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, item, defaultDatabase)}- collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase)case *ast.LikeExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, e.Pattern, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, e.Escape, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, e.Pattern, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, e.Escape, defaultDatabase)case *ast.IsExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)case *ast.ExistsExpr:- collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromNode(result, e.Select, defaultDatabase)case *ast.CastExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)case *ast.ExtractExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)case *ast.IntervalExpr:- collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase)case *ast.CollateExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)case *ast.MatchExpr:for _, column := range e.Columns {- collectOmniAccessTablesFromExpr(result, column, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, column, defaultDatabase)}- collectOmniAccessTablesFromExpr(result, e.Against, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Against, defaultDatabase)case *ast.ConvertExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)case *ast.RowExpr:for _, item := range e.Items {- collectOmniAccessTablesFromExpr(result, item, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, item, defaultDatabase)}case *ast.MemberOfExpr:- collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase, normalizeStarRocksCluster)- collectOmniAccessTablesFromExpr(result, e.Array, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Value, defaultDatabase)+ collectOmniAccessTablesFromExpr(result, e.Array, defaultDatabase)case *ast.ParenExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)case *ast.WeightStringExpr:- collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase, normalizeStarRocksCluster)+ collectOmniAccessTablesFromExpr(result, e.Expr, defaultDatabase)default:}}--func filterStarRocksClusterName(databaseName string) string {- list := strings.Split(databaseName, ":")- if len(list) > 1 {- return list[len(list)-1]- }- return databaseName-}diff --git a/backend/plugin/parser/mysql/query_span_extractor_test.go b/backend/plugin/parser/mysql/query_span_extractor_test.goindex 01f25204cd..198db56183 100644--- a/backend/plugin/parser/mysql/query_span_extractor_test.go+++ b/backend/plugin/parser/mysql/query_span_extractor_test.go@@ -1393,46 +1393,7 @@ func TestOmniQuerySpanScenarioBatch4_CaseSensitivityCoverage(t *testing.T) {})}-func TestOmniQuerySpanScenarioBatch5_StarRocksAndMetadataCoverage(t *testing.T) {- t.Run("starrocks_cluster_qualified_tables", func(t *testing.T) {- gCtx := newOmniStarRocksTestQuerySpanContext()- span, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(- context.Background(),- "SELECT `cluster_name:db`.products.product_info FROM `cluster_name:db`.products",- )- require.NoError(t, err)- require.Equal(t, base.Select, span.Type)- require.Equal(t, []base.QuerySpanResult{- {Name: "product_info", SourceColumns: sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products", Column: "product_info"}}), IsPlainField: true},- }, span.Results)- require.Equal(t, sourceColumnSetFromResources([]base.ColumnResource{{Database: "db", Table: "products"}}), span.SourceColumns)- })-- t.Run("starrocks_system_user_mixing_uses_normalized_database", func(t *testing.T) {- gCtx := newOmniStarRocksTestQuerySpanContext()- _, err := newOmniQuerySpanExtractor("db", gCtx, false).getOmniQuerySpan(- context.Background(),- "SELECT * FROM products, `cluster_name:information_schema`.tables",- )- require.ErrorIs(t, err, base.MixUserSystemTablesError)- })-- t.Run("mysql_and_starrocks_lineage_match_without_cluster_prefix", func(t *testing.T) {- mysqlSpan, err := newOmniQuerySpanExtractor("db", newOmniTestQuerySpanContext(), false).getOmniQuerySpan(- context.Background(),- "SELECT product_info FROM products",- )- require.NoError(t, err)-- starrocksSpan, err := newOmniQuerySpanExtractor("db", newOmniStarRocksTestQuerySpanContext(), false).getOmniQuerySpan(- context.Background(),- "SELECT product_info FROM products",- )- require.NoError(t, err)- require.Equal(t, mysqlSpan.Results, starrocksSpan.Results)- require.Equal(t, mysqlSpan.SourceColumns, starrocksSpan.SourceColumns)- })-+func TestOmniQuerySpanScenarioBatch5_MetadataCoverage(t *testing.T) {t.Run("duplicate_database_case_uses_ignore_case_setting", func(t *testing.T) {gCtx := newOmniCaseCollisionTestQuerySpanContext()@@ -1570,12 +1531,6 @@ func newOmniTestQuerySpanContext() base.GetQuerySpanContext {}}-func newOmniStarRocksTestQuerySpanContext() base.GetQuerySpanContext {- gCtx := newOmniTestQuerySpanContext()- gCtx.Engine = storepb.Engine_STARROCKS- return gCtx-}-func newOmniCaseCollisionTestQuerySpanContext() base.GetQuerySpanContext {dbMetadata := &metadatapb.DatabaseSchemaMetadata{Name: "db",diff --git a/backend/plugin/parser/mysql/query_span_parity_test.go b/backend/plugin/parser/mysql/query_span_parity_test.goindex 150ed77575..3d9938699b 100644--- a/backend/plugin/parser/mysql/query_span_parity_test.go+++ b/backend/plugin/parser/mysql/query_span_parity_test.go@@ -43,9 +43,6 @@ func TestMySQLOmniQuerySpanGoldenHarness(t *testing.T) {total := 0for _, testDataPath := range mysqlOmniProbeFixturePaths {engine := storepb.Engine_MYSQL- if strings.Contains(testDataPath, "starrocks") {- engine = storepb.Engine_STARROCKS- }yamlFile, err := os.Open(testDataPath)if err != nil {diff --git a/backend/plugin/parser/mysql/query_span_probe_test.go b/backend/plugin/parser/mysql/query_span_probe_test.goindex a95c34e2fa..387a944d2b 100644--- a/backend/plugin/parser/mysql/query_span_probe_test.go+++ b/backend/plugin/parser/mysql/query_span_probe_test.go@@ -30,7 +30,6 @@ var mysqlOmniProbeFixturePaths = []string{"test-data/query-span/query_type.yaml","test-data/query-span/standard.yaml","test-data/query-span/case_insensitive.yaml",- "test-data/query-span/starrocks.yaml",}func probeMySQLOmniFixtureParseCoverage(t *testing.T) {diff --git a/backend/plugin/parser/mysql/query_span_test.go b/backend/plugin/parser/mysql/query_span_test.goindex 74b9d68d65..81034c93ff 100644--- a/backend/plugin/parser/mysql/query_span_test.go+++ b/backend/plugin/parser/mysql/query_span_test.go@@ -4,7 +4,6 @@ import ("context""io""os"- "strings""testing"metadatapb "github.com/bytebase/omni/metadata"@@ -37,16 +36,12 @@ func TestGetQuerySpan(t *testing.T) {"test-data/query-span/query_type.yaml","test-data/query-span/standard.yaml","test-data/query-span/case_insensitive.yaml",- "test-data/query-span/starrocks.yaml",})a := require.New(t)for _, testDataPath := range testDataPaths {engine := storepb.Engine_MYSQL- if strings.Contains(testDataPath, "starrocks") {- engine = storepb.Engine_STARROCKS- }testDataPath := testDataPath@@ -175,7 +170,7 @@ func TestExtractTableRefs(t *testing.T) {require.NoError(t, err, "failed to parse statement: %s", test.statement)require.Len(t, parseResult.Items, 1, "expected one parse result for statement: %s", test.statement)- sourceColumns := collectOmniAccessTables(parseResult.Items[0], "db", false)+ sourceColumns := collectOmniAccessTables(parseResult.Items[0], "db")var resources []base.SchemaResourcefor resource := range sourceColumns {resources = append(resources, base.SchemaResource{diff --git a/backend/plugin/parser/mysql/resource_change.go b/backend/plugin/parser/mysql/resource_change.goindex 0109f58852..852c48d36b 100644--- a/backend/plugin/parser/mysql/resource_change.go+++ b/backend/plugin/parser/mysql/resource_change.go@@ -15,8 +15,6 @@ import (func init() {base.RegisterExtractChangedResourcesFunc(storepb.Engine_MYSQL, extractChangedResources)base.RegisterExtractChangedResourcesFunc(storepb.Engine_MARIADB, extractChangedResources)- base.RegisterExtractChangedResourcesFunc(storepb.Engine_STARROCKS, extractChangedResources)- base.RegisterExtractChangedResourcesFunc(storepb.Engine_DORIS, extractChangedResources)}func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {diff --git a/backend/plugin/parser/mysql/test-data/query-span/starrocks.yaml b/backend/plugin/parser/mysql/test-data/query-span/starrocks.yamldeleted file mode 100644index 2fb7bfe85a..0000000000--- a/backend/plugin/parser/mysql/test-data/query-span/starrocks.yaml+++ /dev/null@@ -1,100 +0,0 @@-- description: Test For cluster name- statement: SELECT * FROM v2- defaultDatabase: db- metadata: |-- {- "name": "db",- "schemas": [- {- "name": "",- "tables": [- {- "name": "products",- "columns": [- {- "name": "product_info"- }- ]- }- ],- "views": [- {- "name": "v1",- "definition": "SELECT `cluster_name:db`.`products`.`product_info` FROM `cluster_name:db`.`products`"- },- {- "name": "v2",- "definition": "SELECT `cluster_name:db`.`v1`.`product_info` FROM `cluster_name:db`.`v1`"- }- ]- }- ]- }- querySpan:- type: 1- results:- - name: product_info- sourcecolumns:- - server: ""- database: db- schema: ""- table: products- column: product_info- isplainfield: true- sourcefieldpaths: []- selectasterisk: false- sourcecolumns:- - server: ""- database: db- schema: ""- table: v2- column: ""- predicatecolumns: []-- description: Test For cluster name- statement: SELECT * FROM v1- defaultDatabase: db- metadata: |-- {- "name": "db",- "schemas": [- {- "name": "",- "tables": [- {- "name": "products",- "columns": [- {- "name": "product_info"- }- ]- }- ],- "views": [- {- "name": "v1",- "definition": "SELECT `cluster_name:db`.`products`.`product_info` FROM `cluster_name:db`.`products`"- }- ]- }- ]- }- querySpan:- type: 1- results:- - name: product_info- sourcecolumns:- - server: ""- database: db- schema: ""- table: products- column: product_info- isplainfield: true- sourcefieldpaths: []- selectasterisk: false- sourcecolumns:- - server: ""- database: db- schema: ""- table: v1- column: ""- predicatecolumns: []diff --git a/backend/plugin/parser/starrocks/resource_change.go b/backend/plugin/parser/starrocks/resource_change.gonew file mode 100644index 0000000000..728ebfc6dd--- /dev/null+++ b/backend/plugin/parser/starrocks/resource_change.go@@ -0,0 +1,163 @@+package starrocks++import (+ "strings"++ "github.com/bytebase/omni/starrocks/ast"+ "github.com/pkg/errors"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model"+)++func init() {+ base.RegisterExtractChangedResourcesFunc(storepb.Engine_STARROCKS, extractChangedResources)+}++func extractChangedResources(currentDatabase string, _ string, dbMetadata *model.DatabaseMetadata, asts []base.AST, _ string) (*base.ChangeSummary, error) {+ changedResources := model.NewChangedResources(dbMetadata)++ var dmlCount, insertCount int+ var dmlStatements []string++ addTable := func(name *ast.ObjectName, affectsExistingRows bool) {+ db, table := objectName(name, currentDatabase)+ if table == "" {+ return+ }+ changedResources.AddTable(db, "", &storepb.ChangedResourceTable{Name: table}, affectsExistingRows)+ }++ // addObjectDatabase records a database-only write target for a non-table object DDL+ // (view, materialized view, routine) ONLY when the object's own name carries an+ // explicit database qualifier. An unqualified name records nothing, leaving the+ // request database to stand in. SUP-222.+ addObjectDatabase := func(name *ast.ObjectName) {+ if db, _ := objectName(name, ""); db != "" {+ changedResources.AddDatabase(db)+ }+ }++ for _, unifiedAST := range asts {+ parsed, ok := unifiedAST.(*omniAST)+ if !ok {+ return nil, errors.New("expected omniAST for StarRocks")+ }+ if parsed.node == nil {+ continue+ }++ switch n := parsed.node.(type) {+ case *ast.CreateTableStmt:+ addTable(n.Name, false)++ case *ast.DropTableStmt:+ addTable(n.Name, true)++ case *ast.TruncateTableStmt:+ addTable(n.Target, true)++ case *ast.AlterTableStmt:+ addTable(n.Name, true)++ case *ast.CreateIndexStmt:+ addTable(n.Table, false)++ case *ast.DropIndexStmt:+ addTable(n.Table, false)++ case *ast.LoadDataStmt:+ // One LOAD LABEL carries a data description per target table.+ for _, desc := range n.DataDescs {+ if desc != nil {+ addTable(desc.Target, false)+ }+ }++ case *ast.InsertStmt:+ // INSERT INTO FILES(...) writes to external storage and leaves Target nil,+ // so it records no table while still counting as a DML statement.+ addTable(n.Target, false)+ if len(n.Values) > 0 {+ insertCount += len(n.Values)+ continue+ }+ dmlCount+++ dmlStatements = append(dmlStatements, statementText(parsed))++ case *ast.UpdateStmt:+ // StarRocks updates a single target; FROM names read sources, not write targets.+ addTable(n.Target, false)+ dmlCount+++ dmlStatements = append(dmlStatements, statementText(parsed))++ case *ast.DeleteStmt:+ // USING names read sources, as FROM does for UPDATE.+ addTable(n.Target, false)+ dmlCount+++ dmlStatements = append(dmlStatements, statementText(parsed))++ case *ast.MergeStmt:+ addTable(n.Target, false)+ dmlCount+++ dmlStatements = append(dmlStatements, statementText(parsed))++ // Non-table object DDL: gate by the object's own explicit database qualifier only.+ // CREATE FUNCTION is absent because its AST keeps the name inside RawSignature+ // rather than as an ObjectName, so there is nothing to qualify by.+ case *ast.CreateViewStmt:+ addObjectDatabase(n.Name)+ case *ast.AlterViewStmt:+ addObjectDatabase(n.Name)+ case *ast.DropViewStmt:+ addObjectDatabase(n.Name)+ case *ast.CreateMTMVStmt:+ addObjectDatabase(n.Name)+ case *ast.AlterMTMVStmt:+ addObjectDatabase(n.Name)+ case *ast.DropMTMVStmt:+ addObjectDatabase(n.Name)+ case *ast.DropFunctionStmt:+ addObjectDatabase(n.Name)+ case *ast.CreateProcedureStmt:+ addObjectDatabase(n.Name)+ case *ast.DropProcedureStmt:+ addObjectDatabase(n.Name)++ default:+ }+ }++ return &base.ChangeSummary{+ ChangedResources: changedResources,+ DMLStatements: dmlStatements,+ DMLCount: dmlCount,+ InsertCount: insertCount,+ }, nil+}++// objectName splits an ObjectName into its database and object name. StarRocks writes a+// name as table, db.table or catalog.db.table, so the object is the last part and the+// database the one before it; a catalog qualifier is dropped, since changed resources+// are tracked per database.+func objectName(name *ast.ObjectName, defaultDatabase string) (string, string) {+ if name == nil || len(name.Parts) == 0 {+ return defaultDatabase, ""+ }+ parts := name.Parts+ database := defaultDatabase+ if len(parts) >= 2 {+ database = parts[len(parts)-2]+ }+ return database, parts[len(parts)-1]+}++// statementText returns the text of a statement, ensuring it ends with a semicolon.+func statementText(a *omniAST) string {+ text := strings.TrimSpace(a.text)+ if !strings.HasSuffix(text, ";") {+ text += ";"+ }+ return text+}diff --git a/backend/plugin/parser/starrocks/resource_change_test.go b/backend/plugin/parser/starrocks/resource_change_test.gonew file mode 100644index 0000000000..31b71148e1--- /dev/null+++ b/backend/plugin/parser/starrocks/resource_change_test.go@@ -0,0 +1,149 @@+package starrocks++import (+ "testing"++ "github.com/stretchr/testify/require"++ metadatapb "github.com/bytebase/omni/metadata"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+ "github.com/bytebase/bytebase/backend/plugin/parser/base"+ "github.com/bytebase/bytebase/backend/store/model"+)++// changedTables flattens the built resources into "database.table" strings.+func changedTables(t *testing.T, statement string) []string {+ t.Helper()+ summary := extract(t, statement, nil)+ var result []string+ for _, database := range summary.ChangedResources.Build().GetDatabases() {+ for _, schema := range database.GetSchemas() {+ for _, table := range schema.GetTables() {+ result = append(result, database.GetName()+"."+table.GetName())+ }+ }+ }+ return result+}++func extract(t *testing.T, statement string, dbMetadata *model.DatabaseMetadata) *base.ChangeSummary {+ t.Helper()+ stmts, err := base.ParseStatements(storepb.Engine_STARROCKS, statement)+ require.NoError(t, err)+ summary, err := base.ExtractChangedResources(storepb.Engine_STARROCKS, "db", "", dbMetadata, base.ExtractASTs(stmts), statement)+ require.NoError(t, err)+ return summary+}++func TestExtractChangedResourcesTables(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ want []string+ }{+ {"create table takes the request database", "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"db.t"}},+ {"a qualified name keeps its own database", "CREATE TABLE other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"other.t"}},+ {"a catalog qualifier is dropped", "CREATE TABLE cat.other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{"other.t"}},+ {"drop table", "DROP TABLE t;", []string{"db.t"}},+ {"truncate table", "TRUNCATE TABLE t;", []string{"db.t"}},+ {"alter table", "ALTER TABLE t ADD COLUMN b INT;", []string{"db.t"}},+ {"create index names its table", "CREATE INDEX idx ON t (a);", []string{"db.t"}},+ {"drop index names its table", "DROP INDEX idx ON t;", []string{"db.t"}},+ {"insert names its target", "INSERT INTO t VALUES (1);", []string{"db.t"}},+ {"update names its target, not its source", "UPDATE t SET a = 1 WHERE a IN (SELECT a FROM s);", []string{"db.t"}},+ {"delete names its target, not its source", "DELETE FROM t WHERE a = 1;", []string{"db.t"}},+ {"a view records no table", "CREATE VIEW v AS SELECT 1;", nil},+ } {+ t.Run(tc.name, func(t *testing.T) {+ require.ElementsMatch(t, tc.want, changedTables(t, tc.statement))+ })+ }+}++func TestExtractChangedResourcesDMLAccounting(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ wantDML int+ wantInserts int+ wantTexts []string+ }{+ {name: "insert values counts rows, not statements", statement: "INSERT INTO t VALUES (1), (2), (3);", wantInserts: 3},+ {name: "insert select is a DML statement", statement: "INSERT INTO t SELECT a FROM s;", wantDML: 1, wantTexts: []string{"INSERT INTO t SELECT a FROM s;"}},+ {name: "update is a DML statement", statement: "UPDATE t SET a = 1 WHERE a = 2;", wantDML: 1, wantTexts: []string{"UPDATE t SET a = 1 WHERE a = 2;"}},+ {name: "delete is a DML statement", statement: "DELETE FROM t WHERE a = 1;", wantDML: 1, wantTexts: []string{"DELETE FROM t WHERE a = 1;"}},+ {name: "DDL is neither", statement: "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;"},+ {+ name: "a script accumulates both",+ statement: "INSERT INTO t VALUES (1), (2);\nUPDATE t SET a = 1;\nDELETE FROM t;",+ wantDML: 2,+ wantInserts: 2,+ wantTexts: []string{"UPDATE t SET a = 1;", "DELETE FROM t;"},+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ summary := extract(t, tc.statement, nil)+ require.Equal(t, tc.wantDML, summary.DMLCount)+ require.Equal(t, tc.wantInserts, summary.InsertCount)+ require.Equal(t, tc.wantTexts, summary.DMLStatements)+ })+ }+}++func TestExtractChangedResourcesDatabaseOnlyTargets(t *testing.T) {+ for _, tc := range []struct {+ name string+ statement string+ want []string+ }{+ {"a qualified view records its database", "CREATE VIEW other.v AS SELECT 1;", []string{"other"}},+ {"an unqualified view records nothing", "CREATE VIEW v AS SELECT 1;", []string{}},+ {"alter view", "ALTER VIEW other.v AS SELECT 1;", []string{"other"}},+ {"drop view", "DROP VIEW other.v;", []string{"other"}},+ {"a qualified materialized view records its database", "DROP MATERIALIZED VIEW other.mv;", []string{"other"}},+ {"drop procedure", "DROP PROCEDURE other.p;", []string{"other"}},+ {"a table DDL records no database-only target", "CREATE TABLE other.t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", []string{}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ summary := extract(t, tc.statement, nil)+ require.Equal(t, tc.want, summary.ChangedResources.GetDatabaseOnlyTargets())+ })+ }+}++func TestExtractChangedResourcesAffectedRows(t *testing.T) {+ dbMetadata := model.NewDatabaseMetadata(&metadatapb.DatabaseSchemaMetadata{+ Name: "db",+ Schemas: []*metadatapb.SchemaMetadata{{Tables: []*metadatapb.TableMetadata{{Name: "t", RowCount: 42}}}},+ }, []byte{}, &storepb.DatabaseConfig{}, storepb.Engine_STARROCKS, false /* caseSensitive */)++ for _, tc := range []struct {+ name string+ statement string+ want int64+ }{+ {"drop discards existing rows", "DROP TABLE t;", 42},+ {"truncate discards existing rows", "TRUNCATE TABLE t;", 42},+ {"alter rewrites existing rows", "ALTER TABLE t ADD COLUMN b INT;", 42},+ {"create has no existing rows", "CREATE TABLE t (a INT) DISTRIBUTED BY HASH(a) BUCKETS 1;", 0},+ {"insert leaves existing rows alone", "INSERT INTO t VALUES (1);", 0},+ {"create index leaves existing rows alone", "CREATE INDEX idx ON t (a);", 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ summary := extract(t, tc.statement, dbMetadata)+ require.Equal(t, tc.want, summary.ChangedResources.CountAffectedTableRows())+ })+ }+}++// foreignAST stands in for another engine's AST, which reached this extractor+// through the shared registry before StarRocks had one of its own.+type foreignAST struct{}++func (*foreignAST) ASTStartPosition() *storepb.Position { return nil }++func TestExtractChangedResourcesRejectsForeignAST(t *testing.T) {+ _, err := extractChangedResources("db", "", nil, []base.AST{&foreignAST{}}, "")+ require.ErrorContains(t, err, "expected omniAST for StarRocks")+}diff --git a/backend/plugin/parser/starrocks/starrocks.go b/backend/plugin/parser/starrocks/starrocks.goindex c14dc889eb..0a1bef9860 100644--- a/backend/plugin/parser/starrocks/starrocks.go+++ b/backend/plugin/parser/starrocks/starrocks.go@@ -19,6 +19,7 @@ func init() {// omniAST wraps an omni AST node to implement the base.AST interface.type omniAST struct {node ast.Node+ text stringstartPos *storepb.Position}@@ -67,6 +68,7 @@ func parseStarRocksStatements(statement string) ([]base.ParsedStatement, error)}ps.AST = &omniAST{node: node,+ text: stmt.Text,startPos: stmt.Start,}result = append(result, ps)@@ -103,6 +105,7 @@ func parseStarRocksSQL(statement string) ([]*omniAST, error) {}result = append(result, &omniAST{node: node,+ text: stmt.Text,startPos: stmt.Start,})}diff --git a/backend/server/ultimate.go b/backend/server/ultimate.goindex 139f81a2ee..5f1390dd21 100644--- a/backend/server/ultimate.go+++ b/backend/server/ultimate.go@@ -8,6 +8,7 @@ import (_ "github.com/bytebase/bytebase/backend/plugin/db/cockroachdb"_ "github.com/bytebase/bytebase/backend/plugin/db/cosmosdb"_ "github.com/bytebase/bytebase/backend/plugin/db/databricks"+ _ "github.com/bytebase/bytebase/backend/plugin/db/doris"_ "github.com/bytebase/bytebase/backend/plugin/db/dynamodb"_ "github.com/bytebase/bytebase/backend/plugin/db/elasticsearch"_ "github.com/bytebase/bytebase/backend/plugin/db/hive"