Skip to content

Database governance built for humans and agents — controlling changes and access across every major database.

fix(doris): drop MySQL routine and event probes from the schema dump (#21467)

Vincent Huang59f539dcb79b
diff --git a/backend/plugin/db/starrocks/dump.go b/backend/plugin/db/starrocks/dump.go
index 55b5257118..412404eb78 100644
--- a/backend/plugin/db/starrocks/dump.go
+++ b/backend/plugin/db/starrocks/dump.go
@@ -14,7 +14,6 @@ import (
"github.com/bytebase/bytebase/backend/common"
storepb "github.com/bytebase/bytebase/backend/generated-go/store"
- "github.com/bytebase/bytebase/backend/plugin/db/util"
)
const (
@@ -24,11 +23,6 @@ const (
// Dump and restore.
const (
- settingsStmt = "" +
- "SET character_set_client = %s;\n" +
- "SET character_set_results = %s;\n" +
- "SET collation_connection = %s;\n" +
- "SET sql_mode = '%s';\n"
tableStmtFmt = "" +
"--\n" +
"-- Table structure for `%s`\n" +
@@ -44,11 +38,6 @@ const (
"-- Materialized view structure for `%s`\n" +
"--\n" +
"%s;\n"
- sequenceStmtFmt = "" +
- "--\n" +
- "-- Sequence structure for `%s`\n" +
- "--\n" +
- "%s;\n"
tempViewStmtFmt = "" +
"--\n" +
"-- Temporary view structure for `%s`\n" +
@@ -59,37 +48,12 @@ const (
"-- Temporary view structure for materialized view `%s`\n" +
"--\n" +
"%s\n"
- routineStmtFmt = "" +
- "--\n" +
- "-- %s structure for `%s`\n" +
- "--\n" +
- settingsStmt +
- "DELIMITER ;;\n" +
- "%s ;;\n" +
- "DELIMITER ;\n"
- nullRoutineStmtFmt = "" +
- "--\n" +
- "-- %s structure for `%s`\n" +
- "-- NULL statement because user has insufficient permissions.\n" +
- "--\n"
- eventStmtFmt = "" +
- "--\n" +
- "-- Event structure for `%s`\n" +
- "--\n" +
- settingsStmt +
- "SET time_zone = '%s';\n" +
- "DELIMITER ;;\n" +
- "%s ;;\n" +
- "DELIMITER ;\n"
-
disableUniqueAndForeignKeyCheckStmt = "SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;\nSET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;\n"
restoreUniqueAndForeignKeyCheckStmt = "SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;\nSET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;\n"
)
// Dump dumps the database.
func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {
- // mysqldump -u root --databases dbName --no-data --routines --events --triggers --compact
-
slog.Debug("begin to dump database", slog.String("database", d.databaseName))
db := d.GetDB()
@@ -172,28 +136,6 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSc
}
}
- // Procedure and function (routine) statements.
- routines, err := getRoutines(db, database)
- if err != nil {
- return errors.Wrapf(err, "failed to get routines of database %q", database)
- }
- for _, rt := range routines {
- if _, err := io.WriteString(out, fmt.Sprintf("%s\n", rt.statement)); err != nil {
- return err
- }
- }
-
- // Event statements.
- events, err := getEvents(db, database)
- if err != nil {
- return errors.Wrapf(err, "failed to get events of database %q", database)
- }
- for _, et := range events {
- if _, err := io.WriteString(out, fmt.Sprintf("%s\n", et.statement)); err != nil {
- return err
- }
- }
-
// Restore foreign key check (skipped for StarRocks, which lacks these variables).
if d.dbType != storepb.Engine_STARROCKS {
if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {
@@ -267,19 +209,6 @@ type TableSchema struct {
InvalidView string
}
-// routineSchema describes the schema of a function or procedure (routine).
-type routineSchema struct {
- name string
- routineType string
- statement string
-}
-
-// eventSchema describes the schema of an event.
-type eventSchema struct {
- name string
- statement string
-}
-
// getTables gets all tables of a database using the provided database instance.
func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema, error) {
var tables []*TableSchema
@@ -486,143 +415,3 @@ func getViewColumns(db *sql.DB, dbName, tblName string) ([]string, error) {
}
return fields, nil
}
-
-// getRoutines gets all routines of a database.
-func getRoutines(db *sql.DB, dbName string) ([]*routineSchema, error) {
- var routines []*routineSchema
- for _, routineType := range []string{"FUNCTION", "PROCEDURE"} {
- if err := func() error {
- query := fmt.Sprintf("SHOW %s STATUS WHERE Db = '%s';", routineType, dbName)
- rows, err := db.Query(query)
- if err != nil {
- return errors.Wrapf(err, "failed query %q", query)
- }
- defer rows.Close()
-
- cols, err := rows.Columns()
- if err != nil {
- return err
- }
- var values []any
- for i := 0; i < len(cols); i++ {
- values = append(values, new(any))
- }
- for rows.Next() {
- var r routineSchema
- if err := rows.Scan(values...); err != nil {
- return err
- }
- r.name = fmt.Sprintf("%s", *values[1].(*any))
- r.routineType = fmt.Sprintf("%s", *values[2].(*any))
-
- routines = append(routines, &r)
- }
- return rows.Err()
- }(); err != nil {
- return nil, err
- }
- }
-
- for _, r := range routines {
- stmt, err := getRoutineStmt(db, dbName, r.name, r.routineType)
- if err != nil {
- return nil, errors.Wrapf(err, "failed to call getRoutineStmt(%q, %q, %q)", dbName, r.name, r.routineType)
- }
- r.statement = stmt
- }
- return routines, nil
-}
-
-// getRoutineStmt gets the create statement of a routine.
-func getRoutineStmt(db *sql.DB, dbName, routineName, routineType string) (string, error) {
- query := fmt.Sprintf("SHOW CREATE %s `%s`.`%s`;", routineType, dbName, routineName)
- var sqlmode, charset, collation, unused string
- var stmt sql.NullString
- if err := db.QueryRow(query).Scan(
- &unused,
- &sqlmode,
- &stmt,
- &charset,
- &collation,
- &unused,
- ); err != nil {
- if err == sql.ErrNoRows {
- return "", common.FormatDBErrorEmptyRowWithQuery(query)
- }
- return "", err
- }
- if !stmt.Valid {
- // https://dev.mysql.com/doc/refman/8.0/en/show-create-procedure.html
- slog.Warn("Statement is null, user does not have sufficient permissions",
- slog.String("routineType", routineType),
- slog.String("dbName", dbName),
- slog.String("routineName", routineName))
- return fmt.Sprintf(nullRoutineStmtFmt, getReadableRoutineType(routineType), routineName), nil
- }
- return fmt.Sprintf(routineStmtFmt, getReadableRoutineType(routineType), routineName, charset, charset, collation, sqlmode, stmt.String), nil
-}
-
-// getReadableRoutineType gets the printable routine type.
-func getReadableRoutineType(s string) string {
- switch s {
- case "FUNCTION":
- return "Function"
- case "PROCEDURE":
- return "Procedure"
- default:
- return s
- }
-}
-
-// getEvents gets all events of a database.
-func getEvents(db *sql.DB, dbName string) ([]*eventSchema, error) {
- var events []*eventSchema
- query := fmt.Sprintf("SHOW EVENTS FROM `%s`;", dbName)
- rows, err := db.Query(query)
- if err != nil {
- return nil, err
- }
- defer rows.Close()
-
- cols, err := rows.Columns()
- if err != nil {
- return nil, err
- }
- var values []any
- for i := 0; i < len(cols); i++ {
- values = append(values, new(any))
- }
- for rows.Next() {
- var r eventSchema
- if err := rows.Scan(values...); err != nil {
- return nil, err
- }
- r.name = fmt.Sprintf("%s", *values[1].(*any))
- events = append(events, &r)
- }
- if err := rows.Err(); err != nil {
- return nil, util.FormatErrorWithQuery(err, query)
- }
-
- for _, r := range events {
- stmt, err := getEventStmt(db, dbName, r.name)
- if err != nil {
- return nil, errors.Wrapf(err, "failed to call getEventStmt(%q, %q)", dbName, r.name)
- }
- r.statement = stmt
- }
- return events, nil
-}
-
-// getEventStmt gets the create statement of an event.
-func getEventStmt(db *sql.DB, dbName, eventName string) (string, error) {
- query := fmt.Sprintf("SHOW CREATE EVENT `%s`.`%s`;", dbName, eventName)
- var sqlmode, timezone, stmt, charset, collation, unused string
- if err := db.QueryRow(query).Scan(&unused, &sqlmode, &timezone, &stmt, &charset, &collation, &unused); err != nil {
- if err == sql.ErrNoRows {
- return "", common.FormatDBErrorEmptyRowWithQuery(query)
- }
- return "", err
- }
- return fmt.Sprintf(eventStmtFmt, eventName, charset, charset, collation, sqlmode, timezone, stmt), nil
-}
diff --git a/backend/plugin/db/starrocks/dump_routine_probe_test.go b/backend/plugin/db/starrocks/dump_routine_probe_test.go
new file mode 100644
index 0000000000..092f1cfac5
--- /dev/null
+++ b/backend/plugin/db/starrocks/dump_routine_probe_test.go
@@ -0,0 +1,149 @@
+package starrocks
+
+import (
+ "bytes"
+ "context"
+ "database/sql"
+ "database/sql/driver"
+ "fmt"
+ "io"
+ "regexp"
+ "strings"
+ "sync"
+ "testing"
+
+ "github.com/go-sql-driver/mysql"
+ "github.com/pkg/errors"
+ "github.com/stretchr/testify/require"
+
+ storepb "github.com/bytebase/bytebase/backend/generated-go/store"
+)
+
+// BYT-10211: neither Doris nor StarRocks has MySQL stored routines or events, and Doris
+// kernels that dropped the PL/SQL grammar (SelectDB 4.0.12+/4.1.8+/5.0.5+, apache/doris#58700)
+// reject `SHOW FUNCTION STATUS` outright, which aborted the whole schema sync. The dump must
+// complete without ever sending the MySQL routine or event probes.
+func TestDumpSendsNoRoutineOrEventProbe(t *testing.T) {
+ for _, engine := range []storepb.Engine{storepb.Engine_DORIS, storepb.Engine_STARROCKS} {
+ t.Run(engine.String(), func(t *testing.T) {
+ dumpRecorder.reset(plsqlStrippedCatalog(engine))
+ sqlDB, err := sql.Open(dumpRecorderDriverName, "")
+ require.NoError(t, err)
+ defer sqlDB.Close()
+
+ d := &Driver{dbType: engine, db: sqlDB, databaseName: "prod_tg"}
+ var out bytes.Buffer
+ require.NoError(t, d.Dump(context.Background(), &out, nil))
+
+ require.Contains(t, out.String(), "CREATE TABLE `orders`")
+ require.Contains(t, out.String(), "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`")
+ for _, q := range dumpRecorder.recorded() {
+ require.NotRegexp(t, forbiddenProbe, q)
+ }
+ })
+ }
+}
+
+var (
+ routineStatusProbe = regexp.MustCompile(`(?i)^\s*SHOW\s+(FUNCTION|PROCEDURE)\s+STATUS`)
+ forbiddenProbe = regexp.MustCompile(`(?i)SHOW\s+((FUNCTION|PROCEDURE)\s+STATUS|EVENTS|CREATE\s+(FUNCTION|PROCEDURE|EVENT))`)
+)
+
+// plsqlStrippedCatalog answers the dump's table and view queries with the column shapes a
+// live Doris 4.1.3 returns (and the StarRocks materialized-view catalog in its own shape),
+// and rejects the routine probes with the exact error a PL/SQL-stripped kernel returns.
+func plsqlStrippedCatalog(engine storepb.Engine) func(string) ([]string, [][]driver.Value, error) {
+ return func(query string) ([]string, [][]driver.Value, error) {
+ switch {
+ case routineStatusProbe.MatchString(query):
+ return nil, nil, &mysql.MySQLError{Number: 1105, SQLState: [5]byte{'H', 'Y', '0', '0', '0'}, Message: "errCode = 2, detailMessage = no viable alternative at input 'SHOW FUNCTION'(line 1, pos 5)"}
+ case strings.Contains(query, "information_schema.TABLES"):
+ return []string{"TABLE_NAME", "TABLE_TYPE"}, [][]driver.Value{{"orders", "BASE TABLE"}, {"v_orders", "VIEW"}}, nil
+ case strings.Contains(query, "mv_infos("):
+ return []string{"Name"}, nil, nil
+ case strings.Contains(query, "information_schema.materialized_views"):
+ return []string{"TABLE_NAME", "REFRESH_TYPE"}, nil, nil
+ case strings.HasPrefix(query, "SHOW CREATE TABLE"):
+ return []string{"Table", "Create Table"}, [][]driver.Value{{"orders", "CREATE TABLE `orders` (\n `id` int NULL\n) ENGINE=OLAP"}}, nil
+ case strings.HasPrefix(query, "SHOW CREATE VIEW"):
+ return []string{"View", "Create View", "character_set_client", "collation_connection"}, [][]driver.Value{{"v_orders", "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`", "utf8", "utf8_general_ci"}}, nil
+ case strings.HasPrefix(query, "SHOW COLUMNS FROM"):
+ return []string{"Field", "Type", "Null", "Key", "Default", "Extra"}, [][]driver.Value{{"id", "int", "YES", "", nil, ""}}, nil
+ }
+ return nil, nil, errors.Errorf("unexpected statement for %s: %q", engine, query)
+ }
+}
+
+// dumpRecorder is registered once: database/sql panics on a repeated driver name, which
+// would take the whole package down under `go test -count=2`. Each subtest resets it.
+const dumpRecorderDriverName = "starrocks-dump-recorder"
+
+var dumpRecorder recordingDriver
+
+func init() {
+ sql.Register(dumpRecorderDriverName, &dumpRecorder)
+}
+
+// recordingDriver is a database/sql driver that records every statement and answers it
+// from a canned function, so Dump runs against a scripted server without a network.
+type recordingDriver struct {
+ mu sync.Mutex
+ queries []string
+ answer func(query string) ([]string, [][]driver.Value, error)
+}
+
+func (d *recordingDriver) Open(string) (driver.Conn, error) { return &recordingConn{d: d}, nil }
+
+func (d *recordingDriver) reset(answer func(string) ([]string, [][]driver.Value, error)) {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ d.queries = nil
+ d.answer = answer
+}
+
+func (d *recordingDriver) recorded() []string {
+ d.mu.Lock()
+ defer d.mu.Unlock()
+ return append([]string(nil), d.queries...)
+}
+
+type recordingConn struct{ d *recordingDriver }
+
+func (*recordingConn) Prepare(string) (driver.Stmt, error) {
+ return nil, errors.New("prepare is not supported")
+}
+func (*recordingConn) Close() error { return nil }
+func (*recordingConn) Begin() (driver.Tx, error) { return nil, errors.New("tx is not supported") }
+
+// QueryContext makes database/sql skip Prepare for both db.Query(q) and db.Query(q, arg);
+// the driver interpolates the argument itself, as the real DSN (interpolateParams=true) does.
+func (c *recordingConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {
+ for _, a := range args {
+ query = strings.Replace(query, "?", fmt.Sprintf("'%v'", a.Value), 1)
+ }
+ c.d.mu.Lock()
+ c.d.queries = append(c.d.queries, query)
+ c.d.mu.Unlock()
+ cols, rows, err := c.d.answer(query)
+ if err != nil {
+ return nil, err
+ }
+ return &sliceRows{cols: cols, rows: rows}, nil
+}
+
+type sliceRows struct {
+ cols []string
+ rows [][]driver.Value
+ i int
+}
+
+func (r *sliceRows) Columns() []string { return r.cols }
+func (*sliceRows) Close() error { return nil }
+func (r *sliceRows) Next(dest []driver.Value) error {
+ if r.i >= len(r.rows) {
+ return io.EOF
+ }
+ copy(dest, r.rows[r.i])
+ r.i++
+ return nil
+}