Database governance built for humans and agents — controlling changes and access across every major database.
Go cicddata-governancedata-maskingdata-securitydatabase-accessdatabase-governancedevopsdevsecopsflywaygitopsliquibasemongodbmysqloraclepostgresqlschema-migrationssecuritysql-clientsqlservertidb
fix: LIMIT-aware row estimation path (#20791)
ecmadao5002f5eca4e7
diff --git a/backend/plugin/advisor/mysql/mysql_rules_test.go b/backend/plugin/advisor/mysql/mysql_rules_test.goindex 47942e1a0a..a5a196c845 100644--- a/backend/plugin/advisor/mysql/mysql_rules_test.go+++ b/backend/plugin/advisor/mysql/mysql_rules_test.go@@ -2,6 +2,9 @@ package mysqlimport ("context"+ "database/sql"+ "database/sql/driver"+ "io""testing""github.com/stretchr/testify/require"@@ -13,6 +16,68 @@ import ("github.com/bytebase/bytebase/backend/store/model")+var testMySQLAdvisorExplainRows [][]driver.Value++func init() {+ sql.Register("test_mysql_advisor_explain", testMySQLAdvisorExplainDriver{})+}++type testMySQLAdvisorExplainDriver struct{}++func (testMySQLAdvisorExplainDriver) Open(string) (driver.Conn, error) {+ return testMySQLAdvisorExplainConn{}, nil+}++type testMySQLAdvisorExplainConn struct{}++func (testMySQLAdvisorExplainConn) Prepare(string) (driver.Stmt, error) {+ return nil, driver.ErrSkip+}++func (testMySQLAdvisorExplainConn) Close() error {+ return nil+}++func (testMySQLAdvisorExplainConn) Begin() (driver.Tx, error) {+ return testMySQLAdvisorExplainTx{}, nil+}++func (testMySQLAdvisorExplainConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {+ return &testMySQLAdvisorExplainResultRows{rows: testMySQLAdvisorExplainRows}, nil+}++type testMySQLAdvisorExplainTx struct{}++func (testMySQLAdvisorExplainTx) Commit() error {+ return nil+}++func (testMySQLAdvisorExplainTx) Rollback() error {+ return nil+}++type testMySQLAdvisorExplainResultRows struct {+ rows [][]driver.Value+ idx int+}++func (*testMySQLAdvisorExplainResultRows) Columns() []string {+ return []string{"id", "select_type", "table", "type", "rows", "filtered"}+}++func (*testMySQLAdvisorExplainResultRows) Close() error {+ return nil+}++func (r *testMySQLAdvisorExplainResultRows) Next(dest []driver.Value) error {+ if r.idx >= len(r.rows) {+ return io.EOF+ }+ copy(dest, r.rows[r.idx])+ r.idx+++ return nil+}+func TestMySQLRules(t *testing.T) {rules := []*storepb.SQLReviewRule{{Type: storepb.SQLReviewRule_ENGINE_MYSQL_USE_INNODB, Level: storepb.SQLReviewRule_WARNING},@@ -91,6 +156,53 @@ func TestMySQLRules(t *testing.T) {}}+func TestMySQLRowLimitAdvisorsCapExplainEstimateByLimit(t *testing.T) {+ testMySQLAdvisorExplainRows = [][]driver.Value{+ {int64(1), "SIMPLE", "td", "ALL", int64(1000), "100.00"},+ }+ db, err := sql.Open("test_mysql_advisor_explain", "")+ require.NoError(t, err)+ defer db.Close()++ sm := sheet.NewManager()+ for _, tc := range []struct {+ name string+ statement string+ ruleType storepb.SQLReviewRule_Type+ }{+ {+ name: "affected row limit",+ statement: "UPDATE td SET c = 1 WHERE c = 0 LIMIT 3;",+ ruleType: storepb.SQLReviewRule_STATEMENT_AFFECTED_ROW_LIMIT,+ },+ {+ name: "insert row limit",+ statement: "INSERT INTO td SELECT * FROM source WHERE c = 0 LIMIT 3;",+ ruleType: storepb.SQLReviewRule_STATEMENT_INSERT_ROW_LIMIT,+ },+ } {+ t.Run(tc.name, func(t *testing.T) {+ adviceList, err := advisor.SQLReviewCheck(context.Background(), sm, tc.statement, []*storepb.SQLReviewRule{+ {+ Type: tc.ruleType,+ Level: storepb.SQLReviewRule_WARNING,+ Payload: &storepb.SQLReviewRule_NumberPayload{+ NumberPayload: &storepb.SQLReviewRule_NumberRulePayload{+ Number: 5,+ },+ },+ },+ }, advisor.Context{+ DBType: storepb.Engine_MYSQL,+ Driver: db,+ NoAppendBuiltin: true,+ })+ require.NoError(t, err)+ require.Empty(t, adviceList)+ })+ }+}+func TestMariaDBPriorBackupCheckAdvisor(t *testing.T) {sm := sheet.NewManager()diff --git a/backend/plugin/advisor/mysql/rule_insert_row_limit.go b/backend/plugin/advisor/mysql/rule_insert_row_limit.goindex 69e4742815..9ae5f1c020 100644--- a/backend/plugin/advisor/mysql/rule_insert_row_limit.go+++ b/backend/plugin/advisor/mysql/rule_insert_row_limit.go@@ -88,7 +88,7 @@ func (*InsertRowLimitAdvisor) Check(ctx context.Context, checkCtx advisor.ContexContent: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),StartPosition: common.ConvertANTLRLineToPosition(line),})- } else if rowCount > int64(maxRow) {+ } else if rowCount = capRowsByLimit(rowCount, ins.Select.Limit); rowCount > int64(maxRow) {advice = append(advice, &storepb.Advice{Status: level,Code: code.InsertTooManyRows.Int32(),@@ -128,6 +128,25 @@ func contentStartIndex(text string) int {return 0}+func capRowsByLimit(rowCount int64, limit *ast.Limit) int64 {+ limitCount, ok := limitRowCount(limit)+ if !ok || limitCount >= rowCount {+ return rowCount+ }+ return limitCount+}++func limitRowCount(limit *ast.Limit) (int64, bool) {+ if limit == nil || limit.Count == nil {+ return 0, false+ }+ lit, ok := limit.Count.(*ast.IntLit)+ if !ok || lit.Value < 0 {+ return 0, false+ }+ return lit.Value, true+}+func getInsertRows(res []any) (int64, error) {// the res struct is []any{columnName, columnTable, rowDataList}if len(res) != 3 {diff --git a/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go b/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.goindex b8c366a3ed..511dbe7ccb 100644--- a/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go+++ b/backend/plugin/advisor/mysql/rule_statement_affected_row_limit.go@@ -56,9 +56,12 @@ func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx advcontinue}- // Only handle UPDATE and DELETE statements.- switch node.(type) {- case *ast.UpdateStmt, *ast.DeleteStmt:+ var limit *ast.Limit+ switch stmt := node.(type) {+ case *ast.UpdateStmt:+ limit = stmt.Limit+ case *ast.DeleteStmt:+ limit = stmt.Limitdefault:continue}@@ -87,7 +90,7 @@ func (*StatementAffectedRowLimitAdvisor) Check(ctx context.Context, checkCtx advContent: fmt.Sprintf("failed to get row count for \"%s\": %s", text, err.Error()),StartPosition: common.ConvertANTLRLineToPosition(line),})- } else if rowCount > int64(maxRow) {+ } else if rowCount = capRowsByLimit(rowCount, limit); rowCount > int64(maxRow) {advice = append(advice, &storepb.Advice{Status: level,Code: code.StatementAffectedRowExceedsLimit.Int32(),diff --git a/backend/plugin/db/mysql/stats.go b/backend/plugin/db/mysql/stats.goindex 5639ad8c0f..74c6a7042a 100644--- a/backend/plugin/db/mysql/stats.go+++ b/backend/plugin/db/mysql/stats.go@@ -7,10 +7,12 @@ import ("fmt""strings"+ "github.com/bytebase/omni/mysql/ast""github.com/pkg/errors"storepb "github.com/bytebase/bytebase/backend/generated-go/store""github.com/bytebase/bytebase/backend/plugin/db/util"+ mysqlparser "github.com/bytebase/bytebase/backend/plugin/parser/mysql")func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64, error) {@@ -60,7 +62,7 @@ func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64}if rowsColumn.Valid {- return rowsColumn.Int64, nil+ return capAffectedRowsByLimit(rowsColumn.Int64, statement), nil}}if err := rows.Err(); err != nil {@@ -69,6 +71,43 @@ func (d *Driver) CountAffectedRows(ctx context.Context, statement string) (int64return 0, nil}+func capAffectedRowsByLimit(count int64, statement string) int64 {+ limit, ok := affectedRowsLimit(statement)+ if !ok || limit >= count {+ return count+ }+ return limit+}++func affectedRowsLimit(statement string) (int64, bool) {+ list, err := mysqlparser.ParseMySQLOmni(statement)+ if err != nil || list == nil || len(list.Items) != 1 {+ return 0, false+ }++ var limit *ast.Limit+ switch stmt := list.Items[0].(type) {+ case *ast.UpdateStmt:+ limit = stmt.Limit+ case *ast.DeleteStmt:+ limit = stmt.Limit+ case *ast.InsertStmt:+ if stmt.Select != nil {+ limit = stmt.Select.Limit+ }+ default:+ return 0, false+ }+ if limit == nil || limit.Count == nil {+ return 0, false+ }+ lit, ok := limit.Count.(*ast.IntLit)+ if !ok || lit.Value < 0 {+ return 0, false+ }+ return lit.Value, true+}+func countAffectedRowsForOceanBase(ctx context.Context, sqlDB *sql.DB, dml string) (int64, error) {explainSQL := fmt.Sprintf("EXPLAIN FORMAT=JSON %s", dml)rows, err := sqlDB.QueryContext(ctx, explainSQL)diff --git a/backend/plugin/db/mysql/stats_test.go b/backend/plugin/db/mysql/stats_test.goindex 6c2e9728bf..5a1cb119ae 100644--- a/backend/plugin/db/mysql/stats_test.go+++ b/backend/plugin/db/mysql/stats_test.go@@ -11,9 +11,11 @@ import ()var testOceanBaseExplainRows [][]driver.Value+var testMySQLExplainRows [][]driver.Valuefunc init() {sql.Register("test_oceanbase_explain", testOceanBaseExplainDriver{})+ sql.Register("test_mysql_explain", testMySQLExplainDriver{})}type testOceanBaseExplainDriver struct{}@@ -62,6 +64,89 @@ func (r *testOceanBaseExplainResultRows) Next(dest []driver.Value) error {return nil}+type testMySQLExplainDriver struct{}++func (testMySQLExplainDriver) Open(string) (driver.Conn, error) {+ return testMySQLExplainConn{}, nil+}++type testMySQLExplainConn struct{}++func (testMySQLExplainConn) Prepare(string) (driver.Stmt, error) {+ return nil, driver.ErrSkip+}++func (testMySQLExplainConn) Close() error {+ return nil+}++func (testMySQLExplainConn) Begin() (driver.Tx, error) {+ return nil, driver.ErrSkip+}++func (testMySQLExplainConn) QueryContext(context.Context, string, []driver.NamedValue) (driver.Rows, error) {+ return &testMySQLExplainResultRows{rows: testMySQLExplainRows}, nil+}++type testMySQLExplainResultRows struct {+ rows [][]driver.Value+ idx int+}++func (*testMySQLExplainResultRows) Columns() []string {+ return []string{"id", "select_type", "table", "type", "rows", "filtered"}+}++func (*testMySQLExplainResultRows) Close() error {+ return nil+}++func (r *testMySQLExplainResultRows) Next(dest []driver.Value) error {+ if r.idx >= len(r.rows) {+ return io.EOF+ }+ copy(dest, r.rows[r.idx])+ r.idx+++ return nil+}++func TestCountAffectedRowsCapsExplainEstimateByLimit(t *testing.T) {+ testMySQLExplainRows = [][]driver.Value{+ {int64(1), "SIMPLE", "td", "ALL", int64(1000), "100.00"},+ }+ db, err := sql.Open("test_mysql_explain", "")+ require.NoError(t, err)+ defer db.Close()++ driver := &Driver{db: db}++ for _, tc := range []struct {+ statement string+ want int64+ }{+ {+ statement: "UPDATE td SET c = 1 WHERE c = 0 LIMIT 10;",+ want: 10,+ },+ {+ statement: "DELETE FROM td WHERE c = 0 LIMIT 20;",+ want: 20,+ },+ {+ statement: "INSERT INTO td SELECT * FROM source WHERE c = 0 LIMIT 30;",+ want: 30,+ },+ {+ statement: "UPDATE td SET c = 1 WHERE c = 0;",+ want: 1000,+ },+ } {+ got, err := driver.CountAffectedRows(context.Background(), tc.statement)+ require.NoError(t, err)+ require.Equal(t, tc.want, got, tc.statement)+ }+}+func TestCountAffectedRowsForOceanBaseConcatenatesExplainRows(t *testing.T) {testOceanBaseExplainRows = [][]driver.Value{{`{`},