Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix(doris): drop MySQL routine and event probes from the schema dump (#21467)
Vincent Huang59f539dcb79b
diff --git a/backend/plugin/db/starrocks/dump.go b/backend/plugin/db/starrocks/dump.goindex 55b5257118..412404eb78 100644--- a/backend/plugin/db/starrocks/dump.go+++ b/backend/plugin/db/starrocks/dump.go@@ -14,7 +14,6 @@ import ("github.com/bytebase/bytebase/backend/common"storepb "github.com/bytebase/bytebase/backend/generated-go/store"- "github.com/bytebase/bytebase/backend/plugin/db/util")const (@@ -24,11 +23,6 @@ const (// Dump and restore.const (- settingsStmt = "" +- "SET character_set_client = %s;\n" +- "SET character_set_results = %s;\n" +- "SET collation_connection = %s;\n" +- "SET sql_mode = '%s';\n"tableStmtFmt = "" +"--\n" +"-- Table structure for `%s`\n" +@@ -44,11 +38,6 @@ const ("-- Materialized view structure for `%s`\n" +"--\n" +"%s;\n"- sequenceStmtFmt = "" +- "--\n" +- "-- Sequence structure for `%s`\n" +- "--\n" +- "%s;\n"tempViewStmtFmt = "" +"--\n" +"-- Temporary view structure for `%s`\n" +@@ -59,37 +48,12 @@ const ("-- Temporary view structure for materialized view `%s`\n" +"--\n" +"%s\n"- routineStmtFmt = "" +- "--\n" +- "-- %s structure for `%s`\n" +- "--\n" +- settingsStmt +- "DELIMITER ;;\n" +- "%s ;;\n" +- "DELIMITER ;\n"- nullRoutineStmtFmt = "" +- "--\n" +- "-- %s structure for `%s`\n" +- "-- NULL statement because user has insufficient permissions.\n" +- "--\n"- eventStmtFmt = "" +- "--\n" +- "-- Event structure for `%s`\n" +- "--\n" +- settingsStmt +- "SET time_zone = '%s';\n" +- "DELIMITER ;;\n" +- "%s ;;\n" +- "DELIMITER ;\n"-disableUniqueAndForeignKeyCheckStmt = "SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0;\nSET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0;\n"restoreUniqueAndForeignKeyCheckStmt = "SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS;\nSET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS;\n")// Dump dumps the database.func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSchemaMetadata) error {- // mysqldump -u root --databases dbName --no-data --routines --events --triggers --compact-slog.Debug("begin to dump database", slog.String("database", d.databaseName))db := d.GetDB()@@ -172,28 +136,6 @@ func (d *Driver) Dump(_ context.Context, out io.Writer, _ *metadatapb.DatabaseSc}}- // Procedure and function (routine) statements.- routines, err := getRoutines(db, database)- if err != nil {- return errors.Wrapf(err, "failed to get routines of database %q", database)- }- for _, rt := range routines {- if _, err := io.WriteString(out, fmt.Sprintf("%s\n", rt.statement)); err != nil {- return err- }- }-- // Event statements.- events, err := getEvents(db, database)- if err != nil {- return errors.Wrapf(err, "failed to get events of database %q", database)- }- for _, et := range events {- if _, err := io.WriteString(out, fmt.Sprintf("%s\n", et.statement)); err != nil {- return err- }- }-// Restore foreign key check (skipped for StarRocks, which lacks these variables).if d.dbType != storepb.Engine_STARROCKS {if _, err := io.WriteString(out, restoreUniqueAndForeignKeyCheckStmt); err != nil {@@ -267,19 +209,6 @@ type TableSchema struct {InvalidView string}-// routineSchema describes the schema of a function or procedure (routine).-type routineSchema struct {- name string- routineType string- statement string-}--// eventSchema describes the schema of an event.-type eventSchema struct {- name string- statement string-}-// getTables gets all tables of a database using the provided database instance.func getTables(db *sql.DB, dbName string, dbType storepb.Engine) ([]*TableSchema, error) {var tables []*TableSchema@@ -486,143 +415,3 @@ func getViewColumns(db *sql.DB, dbName, tblName string) ([]string, error) {}return fields, nil}--// getRoutines gets all routines of a database.-func getRoutines(db *sql.DB, dbName string) ([]*routineSchema, error) {- var routines []*routineSchema- for _, routineType := range []string{"FUNCTION", "PROCEDURE"} {- if err := func() error {- query := fmt.Sprintf("SHOW %s STATUS WHERE Db = '%s';", routineType, dbName)- rows, err := db.Query(query)- if err != nil {- return errors.Wrapf(err, "failed query %q", query)- }- defer rows.Close()-- cols, err := rows.Columns()- if err != nil {- return err- }- var values []any- for i := 0; i < len(cols); i++ {- values = append(values, new(any))- }- for rows.Next() {- var r routineSchema- if err := rows.Scan(values...); err != nil {- return err- }- r.name = fmt.Sprintf("%s", *values[1].(*any))- r.routineType = fmt.Sprintf("%s", *values[2].(*any))-- routines = append(routines, &r)- }- return rows.Err()- }(); err != nil {- return nil, err- }- }-- for _, r := range routines {- stmt, err := getRoutineStmt(db, dbName, r.name, r.routineType)- if err != nil {- return nil, errors.Wrapf(err, "failed to call getRoutineStmt(%q, %q, %q)", dbName, r.name, r.routineType)- }- r.statement = stmt- }- return routines, nil-}--// getRoutineStmt gets the create statement of a routine.-func getRoutineStmt(db *sql.DB, dbName, routineName, routineType string) (string, error) {- query := fmt.Sprintf("SHOW CREATE %s `%s`.`%s`;", routineType, dbName, routineName)- var sqlmode, charset, collation, unused string- var stmt sql.NullString- if err := db.QueryRow(query).Scan(- &unused,- &sqlmode,- &stmt,- &charset,- &collation,- &unused,- ); err != nil {- if err == sql.ErrNoRows {- return "", common.FormatDBErrorEmptyRowWithQuery(query)- }- return "", err- }- if !stmt.Valid {- // https://dev.mysql.com/doc/refman/8.0/en/show-create-procedure.html- slog.Warn("Statement is null, user does not have sufficient permissions",- slog.String("routineType", routineType),- slog.String("dbName", dbName),- slog.String("routineName", routineName))- return fmt.Sprintf(nullRoutineStmtFmt, getReadableRoutineType(routineType), routineName), nil- }- return fmt.Sprintf(routineStmtFmt, getReadableRoutineType(routineType), routineName, charset, charset, collation, sqlmode, stmt.String), nil-}--// getReadableRoutineType gets the printable routine type.-func getReadableRoutineType(s string) string {- switch s {- case "FUNCTION":- return "Function"- case "PROCEDURE":- return "Procedure"- default:- return s- }-}--// getEvents gets all events of a database.-func getEvents(db *sql.DB, dbName string) ([]*eventSchema, error) {- var events []*eventSchema- query := fmt.Sprintf("SHOW EVENTS FROM `%s`;", dbName)- rows, err := db.Query(query)- if err != nil {- return nil, err- }- defer rows.Close()-- cols, err := rows.Columns()- if err != nil {- return nil, err- }- var values []any- for i := 0; i < len(cols); i++ {- values = append(values, new(any))- }- for rows.Next() {- var r eventSchema- if err := rows.Scan(values...); err != nil {- return nil, err- }- r.name = fmt.Sprintf("%s", *values[1].(*any))- events = append(events, &r)- }- if err := rows.Err(); err != nil {- return nil, util.FormatErrorWithQuery(err, query)- }-- for _, r := range events {- stmt, err := getEventStmt(db, dbName, r.name)- if err != nil {- return nil, errors.Wrapf(err, "failed to call getEventStmt(%q, %q)", dbName, r.name)- }- r.statement = stmt- }- return events, nil-}--// getEventStmt gets the create statement of an event.-func getEventStmt(db *sql.DB, dbName, eventName string) (string, error) {- query := fmt.Sprintf("SHOW CREATE EVENT `%s`.`%s`;", dbName, eventName)- var sqlmode, timezone, stmt, charset, collation, unused string- if err := db.QueryRow(query).Scan(&unused, &sqlmode, &timezone, &stmt, &charset, &collation, &unused); err != nil {- if err == sql.ErrNoRows {- return "", common.FormatDBErrorEmptyRowWithQuery(query)- }- return "", err- }- return fmt.Sprintf(eventStmtFmt, eventName, charset, charset, collation, sqlmode, timezone, stmt), nil-}diff --git a/backend/plugin/db/starrocks/dump_routine_probe_test.go b/backend/plugin/db/starrocks/dump_routine_probe_test.gonew file mode 100644index 0000000000..092f1cfac5--- /dev/null+++ b/backend/plugin/db/starrocks/dump_routine_probe_test.go@@ -0,0 +1,149 @@+package starrocks++import (+ "bytes"+ "context"+ "database/sql"+ "database/sql/driver"+ "fmt"+ "io"+ "regexp"+ "strings"+ "sync"+ "testing"++ "github.com/go-sql-driver/mysql"+ "github.com/pkg/errors"+ "github.com/stretchr/testify/require"++ storepb "github.com/bytebase/bytebase/backend/generated-go/store"+)++// BYT-10211: neither Doris nor StarRocks has MySQL stored routines or events, and Doris+// kernels that dropped the PL/SQL grammar (SelectDB 4.0.12+/4.1.8+/5.0.5+, apache/doris#58700)+// reject `SHOW FUNCTION STATUS` outright, which aborted the whole schema sync. The dump must+// complete without ever sending the MySQL routine or event probes.+func TestDumpSendsNoRoutineOrEventProbe(t *testing.T) {+ for _, engine := range []storepb.Engine{storepb.Engine_DORIS, storepb.Engine_STARROCKS} {+ t.Run(engine.String(), func(t *testing.T) {+ dumpRecorder.reset(plsqlStrippedCatalog(engine))+ sqlDB, err := sql.Open(dumpRecorderDriverName, "")+ require.NoError(t, err)+ defer sqlDB.Close()++ d := &Driver{dbType: engine, db: sqlDB, databaseName: "prod_tg"}+ var out bytes.Buffer+ require.NoError(t, d.Dump(context.Background(), &out, nil))++ require.Contains(t, out.String(), "CREATE TABLE `orders`")+ require.Contains(t, out.String(), "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`")+ for _, q := range dumpRecorder.recorded() {+ require.NotRegexp(t, forbiddenProbe, q)+ }+ })+ }+}++var (+ routineStatusProbe = regexp.MustCompile(`(?i)^\s*SHOW\s+(FUNCTION|PROCEDURE)\s+STATUS`)+ forbiddenProbe = regexp.MustCompile(`(?i)SHOW\s+((FUNCTION|PROCEDURE)\s+STATUS|EVENTS|CREATE\s+(FUNCTION|PROCEDURE|EVENT))`)+)++// plsqlStrippedCatalog answers the dump's table and view queries with the column shapes a+// live Doris 4.1.3 returns (and the StarRocks materialized-view catalog in its own shape),+// and rejects the routine probes with the exact error a PL/SQL-stripped kernel returns.+func plsqlStrippedCatalog(engine storepb.Engine) func(string) ([]string, [][]driver.Value, error) {+ return func(query string) ([]string, [][]driver.Value, error) {+ switch {+ case routineStatusProbe.MatchString(query):+ return nil, nil, &mysql.MySQLError{Number: 1105, SQLState: [5]byte{'H', 'Y', '0', '0', '0'}, Message: "errCode = 2, detailMessage = no viable alternative at input 'SHOW FUNCTION'(line 1, pos 5)"}+ case strings.Contains(query, "information_schema.TABLES"):+ return []string{"TABLE_NAME", "TABLE_TYPE"}, [][]driver.Value{{"orders", "BASE TABLE"}, {"v_orders", "VIEW"}}, nil+ case strings.Contains(query, "mv_infos("):+ return []string{"Name"}, nil, nil+ case strings.Contains(query, "information_schema.materialized_views"):+ return []string{"TABLE_NAME", "REFRESH_TYPE"}, nil, nil+ case strings.HasPrefix(query, "SHOW CREATE TABLE"):+ return []string{"Table", "Create Table"}, [][]driver.Value{{"orders", "CREATE TABLE `orders` (\n `id` int NULL\n) ENGINE=OLAP"}}, nil+ case strings.HasPrefix(query, "SHOW CREATE VIEW"):+ return []string{"View", "Create View", "character_set_client", "collation_connection"}, [][]driver.Value{{"v_orders", "CREATE VIEW `v_orders` AS SELECT `id` FROM `orders`", "utf8", "utf8_general_ci"}}, nil+ case strings.HasPrefix(query, "SHOW COLUMNS FROM"):+ return []string{"Field", "Type", "Null", "Key", "Default", "Extra"}, [][]driver.Value{{"id", "int", "YES", "", nil, ""}}, nil+ }+ return nil, nil, errors.Errorf("unexpected statement for %s: %q", engine, query)+ }+}++// dumpRecorder is registered once: database/sql panics on a repeated driver name, which+// would take the whole package down under `go test -count=2`. Each subtest resets it.+const dumpRecorderDriverName = "starrocks-dump-recorder"++var dumpRecorder recordingDriver++func init() {+ sql.Register(dumpRecorderDriverName, &dumpRecorder)+}++// recordingDriver is a database/sql driver that records every statement and answers it+// from a canned function, so Dump runs against a scripted server without a network.+type recordingDriver struct {+ mu sync.Mutex+ queries []string+ answer func(query string) ([]string, [][]driver.Value, error)+}++func (d *recordingDriver) Open(string) (driver.Conn, error) { return &recordingConn{d: d}, nil }++func (d *recordingDriver) reset(answer func(string) ([]string, [][]driver.Value, error)) {+ d.mu.Lock()+ defer d.mu.Unlock()+ d.queries = nil+ d.answer = answer+}++func (d *recordingDriver) recorded() []string {+ d.mu.Lock()+ defer d.mu.Unlock()+ return append([]string(nil), d.queries...)+}++type recordingConn struct{ d *recordingDriver }++func (*recordingConn) Prepare(string) (driver.Stmt, error) {+ return nil, errors.New("prepare is not supported")+}+func (*recordingConn) Close() error { return nil }+func (*recordingConn) Begin() (driver.Tx, error) { return nil, errors.New("tx is not supported") }++// QueryContext makes database/sql skip Prepare for both db.Query(q) and db.Query(q, arg);+// the driver interpolates the argument itself, as the real DSN (interpolateParams=true) does.+func (c *recordingConn) QueryContext(_ context.Context, query string, args []driver.NamedValue) (driver.Rows, error) {+ for _, a := range args {+ query = strings.Replace(query, "?", fmt.Sprintf("'%v'", a.Value), 1)+ }+ c.d.mu.Lock()+ c.d.queries = append(c.d.queries, query)+ c.d.mu.Unlock()+ cols, rows, err := c.d.answer(query)+ if err != nil {+ return nil, err+ }+ return &sliceRows{cols: cols, rows: rows}, nil+}++type sliceRows struct {+ cols []string+ rows [][]driver.Value+ i int+}++func (r *sliceRows) Columns() []string { return r.cols }+func (*sliceRows) Close() error { return nil }+func (r *sliceRows) Next(dest []driver.Value) error {+ if r.i >= len(r.rows) {+ return io.EOF+ }+ copy(dest, r.rows[r.i])+ r.i+++ return nil+}